SRETSS/SREBOT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compress the mf (#1267)

Browse Source
This commit is contained in:
NotSoToothless
2026-05-24 19:44:12 -07:00
committed by GitHub
parent 99a43e398e
commit 9c6ca3bcd7
7 changed files with 50 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BOT/autologging.py
+25 -27
View File
@@ -8,6 +8,7 @@ scoreboards, and posts results to configured Discord channels.
# Standard Library Imports
import asyncio
import gzip
import json
import logging
import os
@@ -389,10 +390,10 @@ def minutes_ago(ts, now=None):
def load_replay_data_from_disk(session_id: str):
"""Load replay_data.json from disk for a session."""
"""Load replay_data.json.gz from disk for a session."""
path = replay_data_path(session_id)
if path.is_file():
with open(path, "r", encoding="utf-8") as f:
with gzip.open(path, "rt", encoding="utf-8") as f:
return json.load(f)
return None
@@ -477,13 +478,14 @@ async def process_ws_replays(replays: list[dict]):
# Replay is valid - save to disk
replay_dir = replay_session_dir(hex_id)
replay_dir.mkdir(parents=True, exist_ok=True)
replay_file = replay_dir / "replay_data.json"
replay_file = replay_dir / "replay_data.json.gz"
try:
async with aiofiles.open(replay_file, 'w', encoding='utf-8') as f:
content = json.dumps(local_data, indent=4, ensure_ascii=False)
await f.write(content)
logging.info(f"[WSS] Saved {hex_id} ({len(content)} bytes)")
raw = json.dumps(local_data, ensure_ascii=False).encode("utf-8")
compressed = await asyncio.to_thread(gzip.compress, raw)
async with aiofiles.open(replay_file, "wb") as f:
await f.write(compressed)
logging.info(f"[WSS] Saved {hex_id} ({len(compressed)} bytes compressed)")
except Exception as e:
logging.error(f"[WSS] Failed to save replay {hex_id}: {e}")
continue
@@ -697,8 +699,8 @@ async def build_hex_plus_guild(
replay_path = replay_data_path(sid)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as fp:
replay_data = json.loads(await fp.read())
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except Exception:
logging.error(f"SESSION HEX {sid} FAILED TO GET REPLAY DATA")
mapping[sid] = (g, [])
@@ -958,15 +960,15 @@ async def process_session(
"""
# Load replay JSON
base_dir = replay_session_dir(session_id)
replay_path = base_dir / "replay_data.json"
replay_path = replay_data_path(session_id)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
replay_data = json.loads(await f.read())
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except FileNotFoundError:
logging.error(f"Replay file not found for session ID {session_id}")
return
except json.JSONDecodeError:
logging.error(f"Replay file for session ID {session_id} is invalid JSON")
except (OSError, json.JSONDecodeError) as e:
logging.error(f"Replay file for session ID {session_id} is invalid: {e}")
return
# Extract winner/loser/draw
@@ -1711,17 +1713,16 @@ async def process_comps(new_games):
session_id = game.get('sessionIdHex')
endtime_raw = game.get('endTime')
base_dir = replay_session_dir(session_id)
replay_path = base_dir / "replay_data.json"
replay_path = replay_data_path(session_id)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
replay_data = json.loads(await f.read())
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except FileNotFoundError:
logging.warning(f"(COMP-WRITE) Replay file not found: {replay_path}")
continue
except json.JSONDecodeError:
logging.warning(f"(COMP-WRITE) Invalid JSON in replay: {replay_path}")
except (OSError, json.JSONDecodeError) as e:
logging.warning(f"(COMP-WRITE) Invalid replay: {replay_path}: {e}")
continue
# Translate vehicle names
@@ -1870,15 +1871,12 @@ async def process_stats(new_games):
for sid in sessions_to_process:
endtime_raw = end_times.get(sid, 0)
base_dir = replay_session_dir(sid)
replay_path = base_dir / "replay_data.json"
replay_path = replay_data_path(sid)
try:
async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
replay_data = json.loads(await f.read())
except FileNotFoundError:
continue
except json.JSONDecodeError:
raw = await asyncio.to_thread(replay_path.read_bytes)
replay_data = json.loads(gzip.decompress(raw))
except (FileNotFoundError, OSError, json.JSONDecodeError):
continue
winning_team = replay_data.get("winning_team_squadron")
BOT/botscript.py
+5 -2
View File
@@ -8,6 +8,7 @@ translations, and administrative functions with interactive UI components.
# Standard Library Imports
import asyncio
import gzip
import json
import logging
import math
@@ -4412,8 +4413,10 @@ async def _send_view_match_scoreboard(interaction: discord.Interaction, session_
# Save to disk for buttons (Chat Log, Battle Log read from disk)
replay_dir = replay_session_dir(session_id)
replay_dir.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(replay_dir / "replay_data.json", "w", encoding="utf-8") as f:
await f.write(json.dumps(replay_data, indent=4, ensure_ascii=False))
raw = json.dumps(replay_data, ensure_ascii=False).encode("utf-8")
compressed = await asyncio.to_thread(gzip.compress, raw)
async with aiofiles.open(replay_dir / "replay_data.json.gz", "wb") as f:
await f.write(compressed)
# 3. Translate vehicles
translate = LangTableReader("English")
BOT/render_replay.py
+12 -4
View File
@@ -532,7 +532,11 @@ def _clean_map_key(raw: str) -> str:
def _load_json_file(path: Path) -> dict | None:
try:
return json.loads(path.read_text())
import gzip as _gzip
raw = path.read_bytes()
if path.suffix == ".gz":
raw = _gzip.decompress(raw)
return json.loads(raw)
except Exception as e:
print(f" Failed to parse {path.name}: {e}")
return None
@@ -3279,8 +3283,12 @@ def _convert_local_replay_to_render_dict(replay: dict[str, Any]) -> dict[str, An
def load_gob_file(replay_path: Path) -> dict[str, Any]:
"""Load a replay .json file and normalize it for render/export routines."""
data = json.loads(replay_path.read_text(encoding="utf-8"))
"""Load a replay .json or .json.gz file and normalize it for render/export routines."""
import gzip as _gzip
raw = replay_path.read_bytes()
if replay_path.suffix == ".gz":
raw = _gzip.decompress(raw)
data = json.loads(raw.decode("utf-8"))
if isinstance(data, dict) and {"Players", "Entities", "Mission"}.issubset(data.keys()):
return data
if isinstance(data, dict):
@@ -3585,7 +3593,7 @@ def main():
if args.replay:
replay_path = Path(args.replay)
else:
candidates = sorted(REPLAYS_DIR.glob("*/replay_data.json"))
candidates = sorted(REPLAYS_DIR.glob("*/replay_data.json.gz"))
if not candidates:
candidates = sorted(REPLAYS_DIR.glob("*.json"))
if not candidates:
BOT/task_executors.py
+2 -2
View File
@@ -292,7 +292,7 @@ async def cleanup_replays():
whenever files inside are added or removed (including by this cleanup), which
would otherwise keep dirs perpetually "fresh".
"""
KEEP_FILES = {"replay_data.json"}
KEEP_FILES = {"replay_data.json.gz"}
def _sync_cleanup_replays():
"""Synchronous helper that walks replay dirs and deletes stale files."""
@@ -312,7 +312,7 @@ async def cleanup_replays():
if not entry_path.is_dir():
continue
json_path = entry_path / "replay_data.json"
json_path = entry_path / "replay_data.json.gz"
if json_path.exists():
entry_mtime = json_path.stat().st_mtime
else:
BOT/utils.py
+1 -1
View File
@@ -89,7 +89,7 @@ def replay_session_dir(session_id: str | int) -> Path:
def replay_data_path(session_id: str | int) -> Path:
return replay_session_dir(session_id) / "replay_data.json"
return replay_session_dir(session_id) / "replay_data.json.gz"
# Dev team Discord user IDs (bot owner + trusted devs)
DEV_DISCORD_IDS: set[int] = {
server.js
+3 -3
View File
@@ -27,7 +27,7 @@ const TSS_BATTLES_DB_PATH = path.join(STORAGE_ROOT, 'tss_battles.db');
fs.mkdirSync(REPLAYS_PATH, { recursive: true });
function replayDataPath(sessionId) {
return path.join(REPLAYS_PATH, String(sessionId).toLowerCase(), 'replay_data.json');
return path.join(REPLAYS_PATH, String(sessionId).toLowerCase(), 'replay_data.json.gz');
}
const app = express();
@@ -1443,7 +1443,7 @@ function readReplayJson(sessionId) {
return null;
}
try {
return JSON.parse(fs.readFileSync(replayPath, 'utf-8'));
return JSON.parse(zlib.gunzipSync(fs.readFileSync(replayPath)));
} catch (err) {
log.warn('Failed to read replay JSON', { sessionId, error: err.message });
return null;
@@ -2753,7 +2753,7 @@ app.get('/api/match/:sessionId/replay', (req, res) => {
}
try {
const data = JSON.parse(fs.readFileSync(replayPath, 'utf-8'));
const data = JSON.parse(zlib.gunzipSync(fs.readFileSync(replayPath)));
res.json({
available: true,
session_id: sessionId,
web/server.js
+2 -2
View File
@@ -1828,7 +1828,7 @@ app.get('/api/match/:sessionId/video', async (req, res) => {
const sessionDir = resolveReplaySessionDir(sessionId);
const videoPath = path.join(sessionDir, 'replay_video.mp4');
const replayPath = path.join(sessionDir, 'replay_data.json');
const replayPath = path.join(sessionDir, 'replay_data.json.gz');
// 1. Serve from disk if cached
if (fs.existsSync(videoPath)) {
@@ -1891,7 +1891,7 @@ app.get('/api/match/:sessionId/replay-canvas', async (req, res) => {
const sessionDir = resolveReplaySessionDir(sessionId);
const jsonPath = path.join(sessionDir, 'replay_canvas.json');
const replayPath = path.join(sessionDir, 'replay_data.json');
const replayPath = path.join(sessionDir, 'replay_data.json.gz');
// 1. Serve from disk if cached
if (fs.existsSync(jsonPath)) {