SREBOT/ecosystem.config.js

// Single source of truth for runtime config is SREBOT/.env. Each spawned
// process loads it independently (botscript.py via python-dotenv, server.js
// and web/server.js via `require('dotenv').config()`, github_webhook_updater.py
// via python-dotenv). Do NOT add `env:` blocks below or load dotenv here —
// either would create a second config source that can silently shadow .env.

const DEPLOY_PATH = __dirname;

// Both bots share one venv at BOTS/SHARED/.venv (built from SHARED/requirements.txt).
const PY_INTERPRETER = `${DEPLOY_PATH}/../SHARED/.venv/bin/python`;

// Shared crash-loop governor. Without this, `autorestart` relaunches a process
// that dies on startup forever (every restart_delay). Several apps here share
// SHARED/.env + SHARED/.venv + the STORAGE volume, so one bad shared config can
// make them crash-loop at once and peg all 8 cores until the box is unreachable
// (and `pm2 resurrect` then reproduces it on every boot). With this, PM2 gives
// up after max_restarts attempts that each fail to stay up min_uptime ms,
// marking the app `errored` instead of hammering the CPU. exp_backoff grows the
// delay between attempts (supersedes restart_delay during a crash loop).
const RESTART_POLICY = {
  max_restarts: 10,
  min_uptime: 10000,
  exp_backoff_restart_delay: 200,
};

module.exports = {
  apps: [
    // Discord Bot
    {
      name: 'srebot',
      ...RESTART_POLICY,
      script: 'start_bot.py',
      interpreter: PY_INTERPRETER,
      cwd: DEPLOY_PATH,
      instances: 1,
      autorestart: true,
      watch: false,
      max_memory_restart: '12000M',
      log_file: './logs/bot_combined.log',
      out_file: './logs/bot_out.log',
      error_file: './logs/bot_error.log',
      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
      merge_logs: true,
      kill_timeout: 5000,
      restart_delay: 3000
    },

    // API Server (reads SREBOT_API_PORT from .env)
    {
      name: 'srebot-api',
      ...RESTART_POLICY,
      // Shell wrapper exports UV_THREADPOOL_SIZE at the OS level.
      // PM2's env: and env_file options don't propagate to the child
      // process's OS environ block (required by libuv for threadpool init).
      script: 'start_server.sh',
      interpreter: 'none',
      node_args: [],
      cwd: DEPLOY_PATH,
      instances: 1,
      autorestart: true,
      watch: false,
      max_memory_restart: '4G',
      log_file: './logs/api_combined.log',
      out_file: './logs/api_out.log',
      error_file: './logs/api_error.log',
      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
      merge_logs: true,
      kill_timeout: 5000,
      restart_delay: 2000
    },

    // TSS read-only HTTP API (loopback). Proxied by relay-gateway as /api/tss/*.
    // Reads TSS_API_HOST/PORT from .env (default 127.0.0.1:6100).
    {
      name: 'tssbot-api',
      ...RESTART_POLICY,
      script: PY_INTERPRETER,
      args: '-m web.main',
      interpreter: 'none',
      cwd: `${DEPLOY_PATH}/../TSSBOT`,
      instances: 1,
      autorestart: true,
      watch: false,
      max_memory_restart: '1G',
      log_file: `${DEPLOY_PATH}/logs/tssbot_api_combined.log`,
      out_file: `${DEPLOY_PATH}/logs/tssbot_api_out.log`,
      error_file: `${DEPLOY_PATH}/logs/tssbot_api_error.log`,
      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
      merge_logs: true,
      kill_timeout: 5000,
      restart_delay: 2000
    },

    // Unified relay gateway fronting SREBOT (sqb) and TSSBOT (tss):
    // - Proxies read-only API queries (/api/sre/* -> SREBOT :6000; /api/tss/* -> TSS API or 501)
    // - Streams replay envelopes over /ws/sqb and /ws/tss
    // - Per-key auth (all/sqb/tss) via $STORAGE_VOL_PATH/relay_keys.json
    // Lives in BOTS/SHARED/relay_gateway; loads SREBOT/.env for shared config.
    // Reads SREBOT_EXTERNAL_HOST/PORT/UPSTREAM_URL + STORAGE_VOL_PATH from .env.
    {
      name: 'relay-gateway',
      ...RESTART_POLICY,
      script: PY_INTERPRETER,
      args: '-m relay_gateway.gateway',
      interpreter: 'none',
      cwd: `${DEPLOY_PATH}/../SHARED`,
      instances: 1,
      autorestart: true,
      watch: false,
      max_memory_restart: '1G',
      log_file: `${DEPLOY_PATH}/logs/relay_gateway_combined.log`,
      out_file: `${DEPLOY_PATH}/logs/relay_gateway_out.log`,
      error_file: `${DEPLOY_PATH}/logs/relay_gateway_error.log`,
      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
      merge_logs: true,
      kill_timeout: 5000,
      restart_delay: 2000
    },

    // GitHub Webhook Receiver (auto-deploy on push to main).
    // Reads SREBOT_WEBHOOK_PORT from .env.
    {
      name: 'srebot-webhook',
      ...RESTART_POLICY,
      script: 'github_webhook_updater.py',
      interpreter: PY_INTERPRETER,
      cwd: DEPLOY_PATH,
      instances: 1,
      autorestart: true,
      watch: false,
      max_memory_restart: '500M',
      log_file: './logs/webhook_combined.log',
      out_file: './logs/webhook_out.log',
      error_file: './logs/webhook_error.log',
      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
      merge_logs: true,
      kill_timeout: 3000,
      restart_delay: 2000
    }
  ]
};