SRETSS/SREBOT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compress the mf (#1267)

Browse Source
This commit is contained in:
NotSoToothless
2026-05-24 19:44:12 -07:00
committed by GitHub
parent 99a43e398e
commit 9c6ca3bcd7
7 changed files with 50 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BOT/autologging.py
+25 -27
View File
@@ -8,6 +8,7 @@ scoreboards, and posts results to configured Discord channels.
# Standard Library Imports
import asyncio
import gzip
import json
import logging
import os
@@ -389,10 +390,10 @@ def minutes_ago(ts, now=None):
def load_replay_data_from_disk(session_id: str):
"""Load replay_data.json from disk for a session."""
"""Load replay_data.json.gz from disk for a session."""
path = replay_data_path(session_id)
if path.is_file():
with open(path, "r", encoding="utf-8") as f:
with gzip.open(path, "rt", encoding="utf-8") as f:
return json.load(f)
return None
@@ -477,13 +478,14 @@ async def process_ws_replays(replays: list[dict]):
# Replay is valid - save to disk
replay_dir = replay_session_dir(hex_id)
replay_dir.mkdir(parents=True, exist_ok=True)
replay_file = replay_dir / "replay_data.json"
replay_file = replay_dir / "replay_data.json.gz"
try:
async with aiofiles.open(replay_file, 'w', encoding='utf-8') as f:
content = json.dumps(local_data, indent=4, ensure_ascii=False)
await f.write(content)
logging.info(f"[WSS] Saved {hex_id} ({len(content)} bytes)")
raw = json.dumps(local_data, ensure_ascii=False).encode("utf-8")
compressed = await asyncio.to_thread(gzip.compress, raw)
async with aiofiles.open(replay_file, "wb") as f:
await f.write(compressed)
logging.info(f"[WSS] Saved {hex_id} ({len(compressed)} bytes compressed)")
except Exception as e:
logging.error(f"[WSS] Failed to save replay {hex_id}: {e}")
continue
@@ -697,8 +699,8 @@ async def build_hex_plus_guild(
replay_path = replay_data_path(sid)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as fp:
replay_data = json.loads(await fp.read())
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except Exception:
logging.error(f"SESSION HEX {sid} FAILED TO GET REPLAY DATA")
mapping[sid] = (g, [])
@@ -958,15 +960,15 @@ async def process_session(
"""
# Load replay JSON
base_dir = replay_session_dir(session_id)
replay_path = base_dir / "replay_data.json"
replay_path = replay_data_path(session_id)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
replay_data = json.loads(await f.read())
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except FileNotFoundError:
logging.error(f"Replay file not found for session ID {session_id}")
return
except json.JSONDecodeError:
logging.error(f"Replay file for session ID {session_id} is invalid JSON")
except (OSError, json.JSONDecodeError) as e:
logging.error(f"Replay file for session ID {session_id} is invalid: {e}")
return
# Extract winner/loser/draw
@@ -1711,17 +1713,16 @@ async def process_comps(new_games):
session_id = game.get('sessionIdHex')
endtime_raw = game.get('endTime')
base_dir = replay_session_dir(session_id)
replay_path = base_dir / "replay_data.json"
replay_path = replay_data_path(session_id)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
replay_data = json.loads(await f.read())
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except FileNotFoundError:
logging.warning(f"(COMP-WRITE) Replay file not found: {replay_path}")
continue
except json.JSONDecodeError:
logging.warning(f"(COMP-WRITE) Invalid JSON in replay: {replay_path}")
except (OSError, json.JSONDecodeError) as e:
logging.warning(f"(COMP-WRITE) Invalid replay: {replay_path}: {e}")
continue
# Translate vehicle names
@@ -1870,15 +1871,12 @@ async def process_stats(new_games):
for sid in sessions_to_process:
endtime_raw = end_times.get(sid, 0)
base_dir = replay_session_dir(sid)
replay_path = base_dir / "replay_data.json"
replay_path = replay_data_path(sid)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
replay_data = json.loads(await f.read())
except FileNotFoundError:
continue
except json.JSONDecodeError:
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except (FileNotFoundError, OSError, json.JSONDecodeError):
continue
winning_team = replay_data.get("winning_team_squadron")
BOT/botscript.py
+5 -2
View File
@@ -8,6 +8,7 @@ translations, and administrative functions with interactive UI components.
# Standard Library Imports
import asyncio
import gzip
import json
import logging
import math
@@ -4412,8 +4413,10 @@ async def _send_view_match_scoreboard(interaction: discord.Interaction, session_
# Save to disk for buttons (Chat Log, Battle Log read from disk)
replay_dir = replay_session_dir(session_id)
replay_dir.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(replay_dir / "replay_data.json", "w", encoding="utf-8") as f:
await f.write(json.dumps(replay_data, indent=4, ensure_ascii=False))
raw = json.dumps(replay_data, ensure_ascii=False).encode("utf-8")
compressed = await asyncio.to_thread(gzip.compress, raw)
async with aiofiles.open(replay_dir / "replay_data.json.gz", "wb") as f:
await f.write(compressed)
# 3. Translate vehicles
translate = LangTableReader("English")
BOT/render_replay.py
+12 -4
View File
@@ -532,7 +532,11 @@ def _clean_map_key(raw: str) -> str:
def _load_json_file(path: Path) -> dict | None:
try:
return json.loads(path.read_text())
import gzip as _gzip
raw = path.read_bytes()
if path.suffix == ".gz":
raw = _gzip.decompress(raw)
return json.loads(raw)
except Exception as e:
print(f" Failed to parse {path.name}: {e}")
return None
@@ -3279,8 +3283,12 @@ def _convert_local_replay_to_render_dict(replay: dict[str, Any]) -> dict[str, An
def load_gob_file(replay_path: Path) -> dict[str, Any]:
"""Load a replay .json file and normalize it for render/export routines."""
data = json.loads(replay_path.read_text(encoding="utf-8"))
"""Load a replay .json or .json.gz file and normalize it for render/export routines."""
import gzip as _gzip
raw = replay_path.read_bytes()
if replay_path.suffix == ".gz":
raw = _gzip.decompress(raw)
data = json.loads(raw.decode("utf-8"))
if isinstance(data, dict) and {"Players", "Entities", "Mission"}.issubset(data.keys()):
return data
if isinstance(data, dict):
@@ -3585,7 +3593,7 @@ def main():
if args.replay:
replay_path = Path(args.replay)
else:
candidates = sorted(REPLAYS_DIR.glob("*/replay_data.json"))
candidates = sorted(REPLAYS_DIR.glob("*/replay_data.json.gz"))
if not candidates:
candidates = sorted(REPLAYS_DIR.glob("*.json"))
if not candidates:
BOT/task_executors.py
+2 -2
View File
@@ -292,7 +292,7 @@ async def cleanup_replays():
whenever files inside are added or removed (including by this cleanup), which
would otherwise keep dirs perpetually "fresh".
"""
KEEP_FILES = {"replay_data.json"}
KEEP_FILES = {"replay_data.json.gz"}
def _sync_cleanup_replays():
"""Synchronous helper that walks replay dirs and deletes stale files."""
@@ -312,7 +312,7 @@ async def cleanup_replays():
if not entry_path.is_dir():
continue
json_path = entry_path / "replay_data.json"
json_path = entry_path / "replay_data.json.gz"
if json_path.exists():
entry_mtime = json_path.stat().st_mtime
else:
BOT/utils.py
+1 -1
View File
@@ -89,7 +89,7 @@ def replay_session_dir(session_id: str | int) -> Path:
def replay_data_path(session_id: str | int) -> Path:
return replay_session_dir(session_id) / "replay_data.json"
return replay_session_dir(session_id) / "replay_data.json.gz"
# Dev team Discord user IDs (bot owner + trusted devs)
DEV_DISCORD_IDS: set[int] = {