#!/usr/bin/env python3 """ IkaByte Game Agent v3.7.0 Daemon running on Node machines to manage game servers via Docker. Receives commands from the GamePanel admin panel. v3.7.0 - Anti-DDoS Fortress: deep DDoS binary string analysis (ELF signatures for attack tools), DDoS wrapper/launcher detection (bash/python downloading and running attack tools), botnet C2 detection (IRC bots, Mirai/Gafgyt/ Kaiten variants, C2 beacons, telnet scanning), network flood behavior analysis (packet rate, TX/RX asymmetry, small-packet floods, socket state distribution), DDoS persistence detection (cron, .bashrc, systemd, respawn watchdogs), encoded DDoS payload scanning (base64/hex attack scripts), 100+ DDoS tool process signatures (L3/L4/L7, modern frameworks like MHDDoS, db1000n, UFONet, ddosify), 30+ amplification protocols (DNS, NTP, SSDP, CLDAP, WS-Discovery, CoAP, RDP, Plex, etc.), booter/stresser/C2 domain detection, hardened thresholds (connections 100->60, SYN_SENT 20->10, bandwidth 50->30 MB/s, UDP 50->30), firewall manipulation detection (IP forwarding, RP filter, NAT masquerade), expanded DDoS file patterns. v3.6.0 - Anti-Miner Fortress: deep binary string analysis (ELF strings scan), wrapper/obfuscation detection (bash/python/node wrapping miners), fileless miner detection (memfd_create, /dev/shm, /proc/self/fd), persistence detection (cron, at, systemd, .bashrc, .profile), CPU-to-process correlation (high CPU + unknown binary = miner), encoded config scanning (base64/hex pool URLs and wallet addresses), deep network inspection (DNS queries to mining pools, TLS SNI, proxy/tunnel detection, SOCKS/HTTP proxy to pools), environment variable scanning for hidden pool configs, lowered CPU threshold (70%) with multi-sample variance analysis, expanded miner binary signatures (30+ miners, 50+ pool domains), /proc/[pid]/maps and /proc/[pid]/exe symlink deep inspection. v3.8.0 - File Operations: compress (tar.gz), decompress (zip/tar.gz/tar.bz2/tar.xz/gz), file download with one-time token authentication. v3.5.1 - Auto-Restart & Crash Watchdog: scheduled restarts (every X hours), crash counting with configurable window/max, auto-restart config via POST /api/servers/{uuid}/auto-restart. v3.2.0 - Wrapper Rebuild on Start: rebuilds startup wrapper on every power start/restart, Panel sends startup_command with power actions for config sync, PHP-level npm install injection for Node.js eggs. v3.0.0 - Full Automation Engine: auto-detection of server type (SteamCMD/LinuxGSM/Custom), auto binary discovery, SteamCMD fallback installer, SecurityManager for install script sandboxing, self-healing system, dynamic LD_LIBRARY_PATH, plug & play egg support (no code changes needed for new games). v2.0.0 - Smart Startup: environment validation, binary auto-resolution, dependency checking (ldd), LD_LIBRARY_PATH auto-fix, structured phase logging, post-install validation, post-start health check. Usage: python3 game_agent.py --token TOKEN --port 8443 [--panel-url URL] [--data-dir /var/lib/game-agent] """ import argparse import hashlib import hmac import json import logging import os import re import shlex import signal import socket import ssl import subprocess import sys import threading import time import zipfile from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn from pathlib import Path from urllib.parse import urlparse, parse_qs AGENT_VERSION = "3.8.0" DATA_DIR = "/var/lib/game-agent" LOG_DIR = "/var/log/game-agent" CONSOLE_LOG_LINES = 200 VERIFY_PANEL_SSL = False # Set to True via --verify-panel-ssl flag PANEL_CA_BUNDLE = "" # Path to CA bundle for verifying panel SSL def _get_panel_ssl_context(): """Get SSL context for panel communication, respecting VERIFY_PANEL_SSL setting.""" if not VERIFY_PANEL_SSL: ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE return ctx if PANEL_CA_BUNDLE and os.path.exists(PANEL_CA_BUNDLE): return ssl.create_default_context(cafile=PANEL_CA_BUNDLE) return ssl.create_default_context() # Caches for stats that are expensive to compute every poll _disk_bytes_cache = {} # uuid -> {"bytes": int, "ts": float} _net_baselines = {} # uuid -> {"rx": int, "tx": int, "started_at": str} _download_tokens = {} # token -> {"path": str, "uuid": str, "created": float} DISK_CACHE_TTL = 60 # seconds between du recalculations # --- Security Manager -------------------------------------------- class SecurityManager: """Analyze and sandbox install scripts to prevent malicious operations. Checks install_script content for dangerous patterns BEFORE execution. Logs warnings for suspicious commands, blocks clearly malicious ones. """ BLOCKED_PATTERNS = [ (r'rm\s+(-[a-zA-Z]*\s+)*/$', "Attempt to delete root filesystem"), (r'rm\s+-[a-zA-Z]*r[a-zA-Z]*f[a-zA-Z]*\s+/\s', "Recursive force delete from root"), (r'rm\s+-[a-zA-Z]*f[a-zA-Z]*r[a-zA-Z]*\s+/\s', "Recursive force delete from root"), (r'mkfs\.', "Filesystem format attempt"), (r'dd\s+.*of=/dev/', "Direct disk write attempt"), (r'>\s*/dev/[sh]d', "Direct disk overwrite attempt"), (r'curl\s+[^|]*\|\s*(ba)?sh', "Piping remote content to shell"), (r'wget\s+[^|]*\|\s*(ba)?sh', "Piping remote content to shell"), (r'curl\s+[^|]*\|\s*sudo', "Piping remote content to sudo"), (r'wget\s+[^|]*\|\s*sudo', "Piping remote content to sudo"), (r'chmod\s+[0-7]*[sS]', "Setting SUID/SGID bits"), (r':\(\)\{\s*:\|:&\s*\};:', "Fork bomb detected"), (r'/etc/shadow', "Accessing shadow password file"), (r'/etc/passwd.*>>', "Modifying passwd file"), (r'nc\s+.*-[a-zA-Z]*e', "Netcat reverse shell"), (r'python.*-c.*socket.*connect', "Python reverse shell"), (r'bash\s+-i\s+>&\s*/dev/tcp', "Bash reverse shell"), (r'shutdown|reboot|init\s+[06]|poweroff', "System shutdown/reboot attempt"), ] # Commands that look dangerous but are OK in game server install context ALLOWED_OVERRIDES = [ r'rm\s+-[a-zA-Z]*\s+/tmp/', r'rm\s+-[a-zA-Z]*\s+/mnt/server/', r'rm\s+-[a-zA-Z]*\s+/home/container/', r'curl\s+.*-[oOL]\s', r'curl\s+.*--output', r'wget\s+.*-[oOPq]\s', r'wget\s+.*--output', ] @classmethod def analyze_script(cls, script_content): """Analyze an install script for dangerous patterns. Returns (is_safe: bool, warnings: list[dict]). is_safe=False means the script contains BLOCKED commands. """ if not script_content: return True, [] warnings = [] for line_num, line in enumerate(script_content.split('\n'), 1): stripped = line.strip() if not stripped or stripped.startswith('#'): continue for pattern, description in cls.BLOCKED_PATTERNS: if re.search(pattern, stripped, re.IGNORECASE): is_allowed = any( re.search(ap, stripped, re.IGNORECASE) for ap in cls.ALLOWED_OVERRIDES ) if not is_allowed: warnings.append({ "line": line_num, "content": stripped[:200], "reason": description, }) return len(warnings) == 0, warnings # --- Fraud Detector ---------------------------------------------- class FraudDetector: """Detect abuse and fraud attempts inside game server containers. Scans for: - Cryptocurrency miners (FORTRESS-level detection): * Process name + cmdline + /proc analysis (30+ known miners) * Binary string analysis (strings inside ELF binaries) * Wrapper/obfuscation detection (bash/python/node wrapping miners) * Fileless execution (memfd_create, /dev/shm, /proc/self/fd) * Encoded config scanning (base64/hex pool URLs, wallet addresses) * Miner config file heuristics (JSON pool configs) * CPU pattern correlation with unknown binaries * Persistence mechanisms (cron, at, systemd, .bashrc, .profile) * Deep network inspection (DNS to pools, TLS SNI, proxy tunnels) * Environment variable scanning for hidden pool configs * /proc/[pid]/maps and /proc/[pid]/exe deep analysis * Deleted binary detection (download-run-delete pattern) * LD_PRELOAD rootkit detection (hiding from ps/top) * Kernel thread masquerade detection ([kworker] fakes) - AI/ML workloads (LLMs, training, inference - forbidden on game hosting) - DDoS attacks (FORTRESS-level detection): * Connection flood analysis (TCP outbound, single-target, fan-out) * SYN/UDP/ICMP/GRE flood detection with lowered thresholds * Amplification/reflection attacks (30+ protocols: DNS, NTP, SSDP, CLDAP, etc.) * RAW socket detection (IP spoofing, packet crafting) * Bandwidth spike analysis (TX/RX asymmetry, packet rate) * DDoS binary string analysis (ELF tool signatures even if renamed) * DDoS wrapper/launcher detection (bash/python downloading attack tools) * Botnet C2 detection (IRC bots, Mirai/Gafgyt/Kaiten variants) * Network flood behavior analysis (PPS, traffic asymmetry, small-packet floods) * DDoS persistence (cron, .bashrc, systemd, respawn watchdogs) * Encoded DDoS payload scanning (base64/hex attack scripts) * 100+ DDoS tool process signatures (L3/L4/L7, modern frameworks) * Booter/stresser/C2 domain detection * Firewall manipulation detection (iptables flush, IP forwarding) * Conntrack overflow detection * Process argument analysis (flood flags, attack parameters) - Torrent / P2P clients (transmission, qbittorrent, aria2c) - Reverse shells and C2 beacons - Unauthorized web servers / proxies - Resource abuse patterns (sustained CPU, memory hogging) """ # -- Process name patterns ------------------------------------ MALICIOUS_PROCESS_PATTERNS = [ # -- Cryptocurrency miners (comprehensive) -- (r'\bxmrig\b', "Cryptocurrency miner (XMRig)"), (r'\bcpuminer\b', "Cryptocurrency miner (cpuminer)"), (r'\bethminer\b', "Cryptocurrency miner (ethminer)"), (r'\bphoenixminer\b', "Cryptocurrency miner (PhoenixMiner)"), (r'\bt-rex\b', "Cryptocurrency miner (T-Rex)"), (r'\bnbminer\b', "Cryptocurrency miner (NBMiner)"), (r'\blolminer\b', "Cryptocurrency miner (lolMiner)"), (r'\bgminer\b', "Cryptocurrency miner (GMiner)"), (r'\bminerd\b', "Cryptocurrency miner (minerd)"), (r'\bcgminer\b', "Cryptocurrency miner (cgminer)"), (r'\bbfgminer\b', "Cryptocurrency miner (bfgminer)"), (r'\bccminer\b', "Cryptocurrency miner (ccminer)"), (r'\bsrb[Mm]iner\b', "Cryptocurrency miner (SRBMiner)"), (r'\bteam[Rr]ed[Mm]iner\b', "Cryptocurrency miner (TeamRedMiner)"), (r'\bwildrig\b', "Cryptocurrency miner (WildRig)"), (r'\bnanominer\b', "Cryptocurrency miner (nanominer)"), (r'\bkawpowminer\b', "Cryptocurrency miner (kawpowminer)"), (r'\btonminer\b', "Cryptocurrency miner (TON Miner)"), (r'\bclaymore\b', "Cryptocurrency miner (Claymore)"), (r'\bewbf\b', "Cryptocurrency miner (EWBF)"), (r'\bdero.*miner\b', "Cryptocurrency miner (Dero)"), (r'\bmonero\b.*miner', "Cryptocurrency miner (Monero)"), (r'\bzcash.*miner\b', "Cryptocurrency miner (ZCash)"), (r'\braptoreum\b', "Cryptocurrency miner (Raptoreum)"), (r'\bkaspa.*miner\b', "Cryptocurrency miner (Kaspa)"), # Additional miners and variants (r'\bxmr-?stak\b', "Cryptocurrency miner (XMR-Stak)"), (r'\bmultiminer\b', "Cryptocurrency miner (MultiMiner)"), (r'\bcudo[\s_-]?miner\b', "Cryptocurrency miner (Cudo Miner)"), (r'\bhoneymine\b', "Cryptocurrency miner (HoneyMiner)"), (r'\bkryptex\b', "Cryptocurrency miner (Kryptex)"), (r'\bminergate\b', "Cryptocurrency miner (MinerGate)"), (r'\bnicehashminer\b', "Cryptocurrency miner (NiceHash Miner)"), (r'\bphoenix[\s_-]?miner\b', "Cryptocurrency miner (PhoenixMiner)"), (r'\brillminer\b', "Cryptocurrency miner (RillMiner)"), (r'\bonezermine\b', "Cryptocurrency miner (OneZeroMiner)"), (r'\brigel\b', "Cryptocurrency miner (Rigel)"), (r'\bbzminer\b', "Cryptocurrency miner (BzMiner)"), (r'\bjayddee\b', "Cryptocurrency miner (JayDDee cpuminer-opt)"), (r'\bcpuminer-opt\b', "Cryptocurrency miner (cpuminer-opt)"), (r'\bcpuminer-multi\b', "Cryptocurrency miner (cpuminer-multi)"), (r'\bcpuminer-avx\b', "Cryptocurrency miner (cpuminer AVX)"), (r'\bxmr\b.*\bproxy\b', "Cryptocurrency mining proxy (XMR-Proxy)"), (r'\bxmrig-?proxy\b', "Cryptocurrency mining proxy (XMRig-Proxy)"), # Obfuscated miner variants (common renamed patterns) (r'\b[a-z]{1,3}rig\b', "Possible renamed miner (*rig pattern)"), (r'\bsys(update|service|monitor|kern|init|log)\b', "Suspicious system-impersonating process name"), # Mining pool connection strings in process args (r'stratum\+tcp://', "Mining pool connection (stratum)"), (r'stratum\+ssl://', "Mining pool connection (stratum+ssl)"), (r'stratum2\+tcp://', "Mining pool connection (stratum2)"), (r'--donate-level', "XMRig miner flag"), (r'--algo\s*(cn|rx|gr|kawpow|ethash|randomx|cryptonight)', "Mining algorithm flag"), (r'-o\s+.*pool\.', "Mining pool connection flag"), (r'-o\s+.*:\d{4,5}\b', "Suspicious outbound pool-style connection"), (r'--coin\s+\w+', "Miner coin selection flag"), (r'--url\s+.*stratum', "Miner stratum URL flag"), (r'--user\s+\w+\.\w+', "Miner worker name (wallet.worker)"), (r'--nicehash', "NiceHash miner flag"), (r'--randomx', "RandomX mining algorithm flag"), (r'--cryptonight', "CryptoNight mining algorithm flag"), # Wallet address patterns in process args (r'4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}', "Monero wallet address in process args"), (r'0x[0-9a-fA-F]{40}', "Ethereum wallet address in process args"), (r'bc1[a-zA-HJ-NP-Z0-9]{39,59}', "Bitcoin bech32 wallet address in process args"), (r'[13][a-km-zA-HJ-NP-Z1-9]{25,34}', "Bitcoin legacy wallet address in process args"), (r'--tls\b', "TLS flag (encrypted pool connection)"), (r'--socks[45]?\b', "SOCKS proxy flag (proxy to mining pool)"), (r'--proxy\b.*:\d+', "Proxy connection flag (pool via proxy)"), # -- AI / ML workloads (FORBIDDEN) -- (r'\bollama\b', "AI model server (Ollama)"), (r'\bllama[\._-]?cpp\b', "AI inference engine (llama.cpp)"), (r'\bllama[\._-]?server\b', "AI inference server (llama-server)"), (r'\bvllm\b', "AI inference engine (vLLM)"), (r'\btgi\b.*server', "AI Text Generation Inference server"), (r'\btext-generation-launcher\b', "AI text generation launcher"), (r'\btriton.*server\b', "AI Triton inference server"), (r'\btensorflow.serving\b', "AI TensorFlow Serving"), (r'\btorchserve\b', "AI TorchServe inference server"), (r'\bstable.diffusion\b', "AI image generation (Stable Diffusion)"), (r'\bautomatic1111\b', "AI image generation (AUTOMATIC1111)"), (r'\bcomfyui\b', "AI image generation (ComfyUI)"), (r'\binvokeai\b', "AI image generation (InvokeAI)"), (r'\bwhisper\b.*model', "AI speech recognition (Whisper)"), (r'\bfaster.whisper\b', "AI speech recognition (faster-whisper)"), (r'\bkoboldcpp\b', "AI inference engine (KoboldCpp)"), (r'\bgpt4all\b', "AI model runner (GPT4All)"), (r'\blm[\._-]?studio\b', "AI model runner (LM Studio)"), (r'\blocalai\b', "AI inference server (LocalAI)"), (r'\btabbyapi\b', "AI inference server (TabbyAPI)"), (r'\bexllamav?2?\b', "AI inference engine (ExLlama)"), (r'\boogabooga\b', "AI text-generation-webui (Oobabooga)"), (r'\btransformers\b.*\bpipeline\b', "AI Transformers pipeline"), (r'\btrain\b.*\bmodel\b.*\bepoch', "AI model training detected"), (r'\bpytorch\b.*\bcuda\b', "AI PyTorch GPU workload"), (r'\btensorflow\b.*\bgpu\b', "AI TensorFlow GPU workload"), (r'\bhuggingface\b', "AI HuggingFace model loading"), (r'\bgguf\b', "AI GGUF model format detected"), (r'\bggml\b', "AI GGML model format detected"), # -- DDoS / Network attack tools (FORTRESS-level detection) -- # Layer 7 (Application) DDoS tools (r'\bloic\b', "DDoS tool L7 (LOIC)"), (r'\bhoic\b', "DDoS tool L7 (HOIC)"), (r'\bslowloris\b', "DDoS tool L7 (Slowloris)"), (r'\bgoldeneye\b', "DDoS tool L7 (GoldenEye)"), (r'\bhulk\b.*doser', "DDoS tool L7 (HULK)"), (r'\btorshammer\b', "DDoS tool L7 (Tors Hammer)"), (r'\bxerxes\b', "DDoS tool L3/4 (Xerxes)"), (r'\bpyloris\b', "DDoS tool L7 (PyLoris)"), (r'\br[.-]?u[.-]?dead[.-]?yet\b', "DDoS tool L7 (R-U-Dead-Yet / RUDY)"), (r'\brudy\b.*attack', "DDoS tool L7 (RUDY)"), (r'\bhttpflood\b', "DDoS tool L7 (HTTP Flood)"), (r'\bwreckuest\b', "DDoS tool L7 (Wreckuest)"), (r'\bapache[_-]?killer\b', "DDoS tool L7 (Apache Killer)"), (r'\brange[_-]?header\b.*attack', "DDoS tool L7 (Range Header attack)"), (r'\bhttp[_-]?unbearable[_-]?load\b', "DDoS tool L7 (HTTP Unbearable Load)"), (r'\bslow[_-]?http[_-]?test\b', "DDoS tool L7 (SlowHTTPTest)"), (r'\bslow[_-]?read\b', "DDoS tool L7 (Slow Read attack)"), (r'\bslow[_-]?body\b', "DDoS tool L7 (Slow Body/POST attack)"), (r'\bcc[_-]?attack\b', "DDoS tool L7 (CC Attack / Challenge Collapsar)"), (r'\bhttp[_-]?dos\b', "DDoS tool L7 (HTTP DoS)"), (r'\bhttp[_-]?bomber\b', "DDoS tool L7 (HTTP Bomber)"), (r'\bhttp[_-]?killer\b', "DDoS tool L7 (HTTP Killer)"), (r'\bb0mb3r\b', "DDoS tool L7 (b0mb3r SMS/HTTP bomber)"), (r'\bcfb[_-]?flood\b', "DDoS tool L7 (CloudFlare Bypass Flood)"), (r'\bfloodhttp\b', "DDoS tool L7 (FloodHTTP)"), (r'\btcp[_-]?killer\b', "DDoS tool L7 (TCP Killer)"), (r'\blay7\b.*flood', "DDoS tool L7 (Layer 7 Flood)"), (r'\bl7[_-]?(flood|attack|ddos)\b', "DDoS tool L7 (L7 Attack)"), (r'\bhttp[_-]?get[_-]?flood\b', "DDoS tool L7 (HTTP GET Flood)"), (r'\bhttp[_-]?post[_-]?flood\b', "DDoS tool L7 (HTTP POST Flood)"), (r'\bheadless[_-]?browser[_-]?flood\b', "DDoS tool L7 (Headless Browser DDoS)"), (r'\bpuppeteer.*flood|flood.*puppeteer\b', "DDoS tool L7 (Puppeteer-based flood)"), # Layer 3/4 (Network/Transport) DDoS tools (r'\bhping3?\b', "DDoS tool L3 (hping3 - packet crafter)"), (r'\bstress-ng\b', "Stress testing tool (stress-ng)"), (r'\bnping\b', "Network attack tool (Nping - Nmap packet generator)"), (r'\bscapy\b', "Packet manipulation framework (Scapy)"), (r'\bnetflood\b', "DDoS tool L3 (NetFlood)"), (r'\bsynflood\b', "DDoS tool L4 (SYN Flood)"), (r'\budpflood\b', "DDoS tool L4 (UDP Flood)"), (r'\bicmpflood\b', "DDoS tool L3 (ICMP Flood)"), (r'\bsmurf\b.*attack', "DDoS tool L3 (Smurf Attack)"), (r'\bfraggle\b', "DDoS tool L3 (Fraggle Attack)"), (r'\bland\b.*attack', "DDoS tool L3 (LAND Attack)"), (r'\bteardrop\b', "DDoS tool L3 (Teardrop)"), (r'\bping[_-]?of[_-]?death\b', "DDoS tool L3 (Ping of Death)"), (r'\btcp[_-]?syn[_-]?flood\b', "DDoS tool L4 (TCP SYN Flood)"), (r'\btcp[_-]?ack[_-]?flood\b', "DDoS tool L4 (TCP ACK Flood)"), (r'\btcp[_-]?rst[_-]?flood\b', "DDoS tool L4 (TCP RST Flood)"), (r'\btcp[_-]?fin[_-]?flood\b', "DDoS tool L4 (TCP FIN Flood)"), (r'\budp[_-]?amp\b', "DDoS tool L4 (UDP Amplification)"), (r'\budp[_-]?storm\b', "DDoS tool L4 (UDP Storm)"), (r'\bsyn[_-]?storm\b', "DDoS tool L4 (SYN Storm)"), (r'\back[_-]?flood\b', "DDoS tool L4 (ACK Flood)"), (r'\brst[_-]?flood\b', "DDoS tool L4 (RST Flood)"), (r'\bgre[_-]?flood\b', "DDoS tool L3 (GRE Flood)"), (r'\bip[_-]?fragment\b.*flood', "DDoS tool L3 (IP Fragment Flood)"), (r'\btcp[_-]?null[_-]?flood\b', "DDoS tool L4 (TCP NULL Flood)"), (r'\bxmas[_-]?flood\b', "DDoS tool L4 (XMAS Flood)"), (r'\btcp[_-]?window\b.*attack', "DDoS tool L4 (TCP Window Attack)"), (r'\bssl[_-]?flood\b', "DDoS tool L4 (SSL/TLS Flood)"), (r'\btls[_-]?renegotiation\b.*attack', "DDoS tool L4 (TLS Renegotiation)"), (r'\bthc[_-]?ssl[_-]?dos\b', "DDoS tool L4 (THC-SSL-DoS)"), # Amplification / Reflection attack tools (r'\bdns[_-]?amp\b', "DDoS tool (DNS Amplification)"), (r'\bntp[_-]?amp\b', "DDoS tool (NTP Amplification)"), (r'\bmemcrashed\b', "DDoS tool (Memcached Amplification)"), (r'\bssdp[_-]?(amp|flood|attack)\b', "DDoS tool (SSDP Amplification)"), (r'\bchargen\b.*flood', "DDoS tool (Chargen Amplification)"), (r'\bsnmp[_-]?amp\b', "DDoS tool (SNMP Amplification)"), (r'\bcldap[_-]?amp\b', "DDoS tool (CLDAP Amplification)"), (r'\bripv1[_-]?amp\b', "DDoS tool (RIPv1 Amplification)"), (r'\bldap[_-]?amp\b', "DDoS tool (LDAP Amplification)"), (r'\bws[_-]?discovery[_-]?amp\b', "DDoS tool (WS-Discovery Amplification)"), (r'\bcoap[_-]?amp\b', "DDoS tool (CoAP Amplification)"), (r'\barms[_-]?amp\b', "DDoS tool (ARMS Amplification)"), (r'\brdp[_-]?amp\b', "DDoS tool (RDP Amplification)"), (r'\bjenkins[_-]?amp\b', "DDoS tool (Jenkins UDP Amplification)"), (r'\bquic[_-]?amp\b', "DDoS tool (QUIC Amplification)"), (r'\bdtls[_-]?amp\b', "DDoS tool (DTLS Amplification)"), (r'\bmsdp[_-]?amp\b', "DDoS tool (MSDP Amplification)"), (r'\bplex[_-]?amp\b', "DDoS tool (Plex Amplification / PMSSDP)"), (r'\bmap[_-]?d?\b.*amp', "DDoS tool (mDNS/MAP-D Amplification)"), (r'\breflect(or|ion)[_-]?(flood|attack|ddos)\b', "DDoS tool (Reflection attack)"), # Modern DDoS frameworks and multi-vector tools (r'\bmhddos\b', "DDoS framework (MHDDoS - multi-vector)"), (r'\bdb1000n\b', "DDoS tool (db1000n - Distress)"), (r'\bdistress\b.*ddos', "DDoS tool (Distress)"), (r'\bufonet\b', "DDoS framework (UFONet)"), (r'\bddosify\b', "DDoS tool (ddosify)"), (r'\bddos[_-]?ripper\b', "DDoS tool (DDoS Ripper)"), (r'\bhammer\b.*ddos', "DDoS tool (Hammer)"), (r'\batscan\b', "DDoS tool (ATScan)"), (r'\banonymous[_-]?ddos\b', "DDoS tool (Anonymous DDoS)"), (r'\bbypass[_-]?(ddos|waf|cf|cloudflare)\b', "DDoS Bypass tool (WAF/CF bypass)"), (r'\bcf[_-]?bypass\b', "DDoS tool (CloudFlare bypass)"), (r'\bwaf[_-]?bypass\b.*flood', "DDoS tool (WAF bypass flood)"), (r'\bazure[_-]?ddos\b', "DDoS tool targeting Azure"), (r'\boverflow\b.*ddos', "DDoS overflow tool"), (r'\bpentagon\b.*ddos', "DDoS tool (Pentagon DDoS)"), (r'\bvenus\b.*ddos', "DDoS tool (Venus DDoS)"), (r'\bsocket[_-]?stress\b', "DDoS tool (Socket Stress)"), (r'\bl4[_-]?(flood|attack|ddos)\b', "DDoS tool L4 (L4 Attack)"), (r'\bl3[_-]?(flood|attack|ddos)\b', "DDoS tool L3 (L3 Attack)"), # Booters / Stressers / Botnet tools (r'\bmirai\b', "Botnet malware (Mirai)"), (r'\bgafgyt\b', "Botnet malware (Gafgyt/Bashlite)"), (r'\bbashlite\b', "Botnet malware (Bashlite)"), (r'\bqbot\b', "Botnet malware (QBot)"), (r'\bircbot\b', "IRC Botnet controller"), (r'\bkaiten\b', "IRC DDoS Bot (Kaiten)"), (r'\bstresser\b', "DDoS Stresser/Booter service"), (r'\bbooter\b', "DDoS Booter service"), (r'\braft[_-]?flood\b', "DDoS tool (RAFT Flood)"), (r'\bblack[_-]?nurse\b', "DDoS tool (BlackNurse ICMP)"), (r'\btsuname\b', "Botnet malware (Tsunami/Kaiten variant)"), (r'\bhajime\b', "Botnet malware (Hajime IoT worm)"), (r'\bmozi\b.*bot', "Botnet malware (Mozi)"), (r'\bsatori\b', "Botnet malware (Satori/Okiru)"), (r'\bokiru\b', "Botnet malware (Okiru)"), (r'\bsylveon\b', "Botnet malware (Sylveon)"), (r'\bbillgates\b', "Botnet malware (BillGates DDoS)"), (r'\bxor[._-]?ddos\b', "Botnet malware (XOR.DDoS)"), (r'\bchinaz\b', "Botnet malware (ChinaZ DDoS)"), (r'\bnitol\b', "Botnet malware (Nitol)"), (r'\bdofloo\b', "Botnet malware (DoFloo/AES.DDoS)"), (r'\baes[._-]?ddos\b', "Botnet malware (AES.DDoS)"), (r'\bhandymenny\b', "Botnet malware (HandyMenny)"), (r'\bircflood\b', "IRC flood bot"), (r'\bslow[_-]?dns\b', "DNS tunneling tool (SlowDNS)"), (r'\biodine\b', "DNS tunneling tool (iodine)"), (r'\bdnscat\b', "DNS tunneling C2 (dnscat2)"), (r'\bchisel\b.*tunnel', "Network tunneling tool (Chisel)"), # Packet crafting & network manipulation (r'\bnemesis\b', "Packet injection tool (Nemesis)"), (r'\bpacketh\b', "Packet manipulation tool (packETH)"), (r'\byersinia\b', "Network attack tool (Yersinia - L2 attacks)"), (r'\bmacof\b', "MAC flooding tool (macof)"), (r'\bettercap\b', "Network attack framework (Ettercap/Bettercap)"), (r'\barpspoof\b', "ARP spoofing tool (arpspoof)"), (r'\btcpreplay\b', "Packet replay tool (tcpreplay)"), (r'\bpacketsender\b', "Packet sender tool"), (r'\bsockstress\b', "DDoS tool (Sockstress)"), (r'\bnmap\b.*-sS.*--max-rate', "Nmap SYN scan with high rate (scanning/flood)"), (r'\bmasscan\b', "Mass port scanner (masscan)"), (r'\bzmap\b', "Internet-wide scanner (ZMap)"), (r'\brusscan\b', "Fast scanner (RustScan)"), # Generic flood patterns in command arguments (r'--flood\b', "Flood flag in command arguments"), (r'\b-{1,2}(syn|ack|rst|fin|udp|icmp)[_-]?flood\b', "Protocol flood flag detected"), (r'\bfork[_-]?bomb\b', "Fork bomb detected"), (r'\b:(\){\s*:\|:&\s*\};:', "Bash fork bomb"), (r'--attack[_-]?(method|type|mode)\b', "DDoS attack method flag"), (r'--(target|victim|host)\b.*:\d+', "DDoS target specification"), (r'--threads?\s+\d{2,}\b', "High thread count (DDoS indicator)"), (r'\b-t\s+\d{3,}\s', "Very high thread count flag"), (r'--duration\s+\d+', "Attack duration flag"), (r'--packets?\s+\d{4,}', "High packet count flag"), (r'--connections?\s+\d{3,}', "High connection count flag"), (r'--rpc\s+\d{2,}', "High requests-per-connection (DDoS flag)"), # -- Torrent / P2P -- (r'\btransmission-daemon\b', "Torrent client (Transmission)"), (r'\bqbittorrent\b', "Torrent client (qBittorrent)"), (r'\bdeluge[d]?\b', "Torrent client (Deluge)"), (r'\baria2c?\b', "Download/torrent client (aria2)"), (r'\brtorrent\b', "Torrent client (rTorrent)"), # -- Reverse shells / C2 -- (r'\bmetasploit\b', "Exploitation framework (Metasploit)"), (r'\bmsfconsole\b', "Exploitation framework (msfconsole)"), (r'\bmsfvenom\b', "Payload generator (msfvenom)"), (r'\bcobalt.*strike\b', "C2 framework (Cobalt Strike)"), (r'\bsocat\b.*\bexec\b', "Reverse shell via socat"), # -- Unauthorized services -- (r'\bnginx\b', "Unauthorized web server (nginx)"), (r'\bapache2?\b', "Unauthorized web server (Apache)"), (r'\bsquid\b', "Unauthorized proxy (Squid)"), (r'\bprivoxy\b', "Unauthorized proxy (Privoxy)"), (r'\btor\b\s', "Tor anonymization service"), (r'\bopenvpn\b', "Unauthorized VPN server"), (r'\bwireguard\b', "Unauthorized VPN server (WireGuard)"), ] # -- File patterns -------------------------------------------- MALICIOUS_FILE_PATTERNS = [ # Crypto miners "xmrig", "cpuminer", "minerd", "ethminer", "phoenixminer", "nbminer", "lolminer", "t-rex", "gminer", "cgminer", "bfgminer", "ccminer", "srbminer", "teamredminer", "wildrig", "nanominer", "kawpowminer", "claymore", "ewbf", "tonminer", "config.json", # common miner config (checked with content heuristics) ".bashrc_miner", "miner.sh", "start_miner", "mine.sh", "automine", "cryptonight", "randomx", # Obfuscated miner filenames (common disguises) "xmr-stak", "cpuminer-opt", "cpuminer-multi", "cpuminer-avx", "xmrig-proxy", "bzminer", "rigel", "onezerominer", "pool_config", "pool.txt", "wallet.txt", "mining.conf", ".miner_config", ".pool_url", ".xmr", "config_miner", "miner_config", "hashrate", # DDoS (FORTRESS-level file detection) "loic", "hoic", "slowloris.py", "goldeneye.py", "torshammer", "xerxes", "hulk.py", "pyloris", "rudy", "httpflood", "synflood", "udpflood", "icmpflood", "netflood", "dns_amp", "ntp_amp", "memcrashed", "mirai", "gafgyt", "bashlite", "stresser", "booter", "ddos", "flood.py", "flood.sh", "apache_killer", "blacknurse", "smurf", "mhddos", "db1000n", "ufonet", "ddosify", "ddos_ripper", "atscan", "cfb_flood", "l7flood", "l4flood", "l3flood", "tcp_flood", "syn_flood", "udp_flood", "icmp_flood", "http_flood", "http_bomber", "http_killer", "cc_attack", "slowhttptest", "slow_read", "slow_body", "amplifier", "reflector", "spoofer", "ddos_attack", "attack.py", "attack.sh", "attack.js", "attack.go", "attack.c", "flood.rb", "flood.pl", "flood.js", "flood.go", "flood.c", "stress.py", "stress.sh", "bot.py", "bot.sh", "botnet", "kaiten", "tsunami", "hajime", "satori", "okiru", "billgates", "xor_ddos", "chinaz", "nitol", "dofloo", "ircbot", "irc_flood", "qbot", "sockstress", "thc_ssl_dos", "ssl_flood", "tls_flood", "dns_tunnel", "dnscat", "iodine", "slowdns", "hping_flood", "nping_flood", "scapy_flood", "packet_flood", "raw_socket", "ip_spoof", # AI/ML model files (large - these should never be on game servers) ".gguf", ".ggml", ".safetensors", ".bin.index.json", "pytorch_model.bin", "model.safetensors", "adapter_model", "tokenizer.model", "tokenizer.json", ] # -- AI/ML specific file extensions (large model files) ------- AI_MODEL_EXTENSIONS = { ".gguf", ".ggml", ".safetensors", ".onnx", ".pt", ".pth", ".h5", ".hdf5", ".tflite", ".pb", ".mlmodel", } # Size threshold for AI model detection (500MB+ = likely a model) AI_MODEL_SIZE_THRESHOLD = 500 * 1024 * 1024 # 500MB # AI/ML Python packages that shouldn't be installed in game containers AI_PYTHON_PACKAGES = [ "torch", "tensorflow", "transformers", "diffusers", "accelerate", "bitsandbytes", "auto-gptq", "exllama", "exllamav2", "llama-cpp-python", "vllm", "triton", "onnxruntime", "sentencepiece", "tokenizers", "safetensors", "huggingface-hub", "optimum", "trl", "peft", "ctransformers", "gpt4all", "langchain", "openai", ] # -- Miner config content heuristics -------------------------- MINER_CONFIG_PATTERNS = [ r'"algo"\s*:', r'"pool"\s*:', r'"wallet"\s*:', r'stratum\+tcp', r'stratum\+ssl', r'"coin"\s*:', r'"donate-level"\s*:', r'"nicehash"\s*:', r'"mining"\s*:', r'"threads"\s*:.*\d+', r'"cpu"\s*:\s*\{', r'"pools"\s*:\s*\[', r'"url"\s*:\s*".*:\d{4,5}"', r'"user"\s*:\s*"[a-zA-Z0-9]{30,}"', # long wallet address r'"pass"\s*:\s*"x"', # typical miner "password" = "x" r'"rig-id"\s*:', r'"worker"\s*:', # Extended patterns for obfuscated configs r'"hashrate"\s*:', r'"max-cpu-usage"\s*:', r'"background"\s*:\s*true', r'"syslog"\s*:', r'"log-file"\s*:', r'"print-time"\s*:', r'"retries"\s*:', r'"retry-pause"\s*:', r'"keepalive"\s*:\s*true', r'"tls"\s*:\s*true', r'"coin"\s*:\s*"(XMR|ETH|BTC|RVN|ERG|KAS|DERO|RTM|ZEPH)"', r'"daemon"\s*:\s*true', # miner daemon mode r'4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}', # Monero wallet in config r'0x[0-9a-fA-F]{40}', # Ethereum wallet in config ] # -- Known mining pool domains -------------------------------- MINING_POOL_DOMAINS = [ "pool.", "mining.", "nicehash", "nanopool", "hashvault", "minexmr", "f2pool", "ethermine", "flypool", "2miners", "unmineable", "supportxmr", "moneroocean", "herominers", "miningpoolhub", "prohashing", "zpool", "zergpool", "woolypooly", "kryptex", "cruxpool", "crazypool", "hashrate", "minergate", "viabtc", "antpool", "poolin", "luxor", "braiins", "slushpool", "kaspapool", "acc-pool", "dero.herominers", # Extended pool domains "xmrpool", "hashxmr", "c3pool", "monerod", "rx.unmineable", "randomxmonero", "gulf.moneroocean", "auto.skypool", "pool.rplant", "pool.hashvault", "stratum.", "stratumserver", "monerop", "xmrvsbeast", "p2pool", "mini.p2pool", "pool.kryptex", "usxmrpool", "pool.supportxmr", "pool.xmr", "pool.minexmr", "pool.hashvault", "bohemianpool", "haven.herominers", "xmr.pool.minergate", "xmrfast", "miningrigrentals", "zephyrpool", "pool.zephyrprotocol", "kaspa.acc-pool", "kas.pool", "flux.runonflux", "ravencoin.flypool", "ergo.herominers", "ton.pool", "tonpool", "toncoinpool", "pool.binance", "pool.btc.com", "emcd.io", ] # -- Strings found inside miner ELF binaries -------------------- # These strings appear inside compiled miner binaries and cannot # be hidden by renaming the file or wrapping in a shell script MINER_BINARY_STRINGS = [ b"stratum+tcp://", b"stratum+ssl://", b"stratum2+tcp://", b"stratum+tls://", b"--donate-level", b"--donate-over-proxy", b"xmrig", b"XMRig", b"XMRIG", b"cpuminer", b"minerd", b"cgminer", b"bfgminer", b"randomx_vm", b"randomx_calculate", b"rx/0", b"rx/wow", b"rx/arq", b"rx/sfx", b"rx/keva", b"cn/0", b"cn/1", b"cn/2", b"cn/r", b"cn/fast", b"cn-lite", b"cn-heavy", b"cn-pico", b"cryptonight", b"CryptoNight", b"ghostrider", b"GhostRider", b"kawpow", b"ethash", b"etchash", b"progpow", b"randomx", b"RandomX", b"RANDOMX", b"hashrate", b"Hashrate", b"HASHRATE", b"h/s", b"hash/s", b"H/s", b"kH/s", b"MH/s", b"GH/s", b"pool_address", b"wallet_address", b"mining-pool", b"miner_id", b"rig_id", b"hugepages", b"huge-pages", b"1gb-pages", b"donate-level", b"donate_level", b"--algo=", b"--coin=", b"--url=", b"Solo mining", b"solo+tcp://", b"Job received", b"new job from", b"submit accepted", b"share accepted", b"login.*agent.*xmrig", b"\"blob\"", b"\"job_id\"", b"\"target\"", # stratum JSON-RPC ] # -- Wallet address regex patterns (for binary/text scanning) -- WALLET_PATTERNS = [ (r'4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}', "Monero (XMR) wallet"), (r'8[0-9AB][1-9A-HJ-NP-Za-km-z]{93}', "Monero (XMR) subaddress"), (r'0x[0-9a-fA-F]{40}', "Ethereum (ETH) wallet"), (r'bc1[a-zA-HJ-NP-Z0-9]{39,59}', "Bitcoin bech32 (BTC) wallet"), (r'[13][a-km-zA-HJ-NP-Z1-9]{25,34}', "Bitcoin legacy (BTC) wallet"), (r'kaspa:[a-z0-9]{61,63}', "Kaspa (KAS) wallet"), (r't1[a-zA-Z0-9]{33}', "ZCash (ZEC) wallet"), (r'D[a-km-zA-HJ-NP-Z1-9]{33}', "Dogecoin (DOGE) wallet"), (r'L[a-km-zA-HJ-NP-Z1-9]{33}', "Litecoin (LTC) wallet"), (r'r[a-km-zA-HJ-NP-Z1-9]{33}', "Raptoreum (RTM) wallet"), (r'dero1[a-z0-9]{60,66}', "Dero (DERO) wallet"), (r'9[0-9AB][1-9A-HJ-NP-Za-km-z]{93}', "Monero integrated address"), (r'UQ[a-zA-Z0-9_-]{46}', "TON wallet"), (r'EQ[a-zA-Z0-9_-]{46}', "TON wallet"), ] # -- Strings found inside DDoS tool ELF binaries -------------- # These strings appear inside compiled attack tools and cannot # be hidden by renaming the file or wrapping in a shell script DDOS_BINARY_STRINGS = [ # Flood-related strings b"flood_start", b"flood_stop", b"start_flood", b"stop_flood", b"syn_flood", b"udp_flood", b"tcp_flood", b"icmp_flood", b"http_flood", b"dns_flood", b"ntp_flood", b"ssdp_flood", b"SOCK_RAW", b"IPPROTO_RAW", b"IPPROTO_ICMP", b"IPPROTO_TCP", b"IP_HDRINCL", b"setsockopt", b"sendto", b"--flood", b"--rand-source", b"--syn", b"--ack", b"packet_size", b"pkt_size", b"payload_size", b"attack_method", b"attack_type", b"attack_duration", b"target_ip", b"target_port", b"target_host", b"victim_ip", b"victim_port", b"destination_ip", b"threads_count", b"thread_count", b"num_threads", b"packets_sent", b"packets_per_second", b"pps", b"amplification", b"reflection", b"spoofed", b"C2_SERVER", b"c2_server", b"cnc_server", b"command_and_control", b"botnet", b"zombie", b"bot_count", # Mirai signatures b"/bin/busybox", b"MIRAI", b"mirai", b"scanner_init", b"attack_init", b"killer_init", b"table_init", b"attack_tcp_syn", b"attack_tcp_ack", b"attack_tcp_stomp", b"attack_udp_generic", b"attack_udp_vse", b"attack_udp_dns", b"attack_gre_ip", b"attack_gre_eth", # Gafgyt/Bashlite signatures b"GAFGYT", b"gafgyt", b"BASHLITE", b"bashlite", b"JUNK", b"UDP", b"TCP", b"HOLD", b"LOLNOGTFO", # Generic DDoS tool strings b"Usage: ", b"attack", b" -p ", b"port", b"Sending packets", b"Flooding", b"flooding", b"DoS attack", b"DDoS attack", b"denial of service", b"Layer 7", b"Layer 4", b"Layer 3", b"HTTP GET", b"HTTP POST", b"slowloris", b"connection_timeout", b"socket_timeout", b"random_ip", b"randomize_ip", b"spoof_ip", b"dns_amplify", b"ntp_monlist", b"memcached_amp", b"raw_socket", b"raw socket", b"hping", b"nping", b"scapy", b"--attack-method", b"--attack-type", b"net.ipv4.ip_forward", b"ip_forward", ] # -- Known booter/stresser/C2 domains ------------------------- # These domains are associated with DDoS-for-hire and botnet C2 BOOTER_C2_DOMAINS = [ # Generic DDoS infrastructure patterns "stresser.", "booter.", "ddos.", "stress.", "boot.", "attack.", "api.stress", "api.boot", "api.ddos", # Known C2/botnet infrastructure "cnc.", "c2.", "cc.", "command.", "control.", "botnet.", "zombie.", "bot.", "panel.ddos", # IRC C2 channels (common for Mirai/Gafgyt) "irc.", "undernet.", "efnet.", "dalnet.", "quakenet.", "rizon.", "oftc.", # Common DDoS API patterns "api.stresser", "api.booter", "stressapi.", "bootapi.", "ddosapi.", # Proxy/anonymization used by DDoS "socks4.", "socks5.", "proxy-list.", "proxylist.", "freeproxy.", ] # -- DDoS script content patterns (for deep file scanning) ---- DDOS_SCRIPT_PATTERNS = [ r'import\s+socket.*\bsendto\b', # Raw UDP flood in Python r'socket\.SOCK_RAW', # Raw socket in Python r'socket\.IPPROTO_RAW', # Raw IP protocol r'socket\.IPPROTO_ICMP', # ICMP flood r'hping3?\s+.*--flood', # hping flood command r'for\s.*in\s.*range.*socket.*send', # Loop-based flood r'while\s+true.*socket.*send', # Infinite loop flood r'threading.*target.*flood', # Multi-threaded flood r'asyncio.*flood', # Async flood r'multiprocessing.*flood', # Multi-process flood r'subprocess.*hping|subprocess.*nping', # Spawning attack tools r'sock\.sendto\(.*\*\s*\d{3,}', # Sending multiplied data r'amplif(y|ication)', # Amplification attack script r'reflect(or|ion)', # Reflection attack script r'spoof.*ip|ip.*spoof', # IP spoofing r'syn\s*flood|synflood', # SYN flood script r'udp\s*flood|udpflood', # UDP flood script r'tcp\s*flood|tcpflood', # TCP flood script r'http\s*flood|httpflood', # HTTP flood script r'slowloris|slow\s*read', # Slow-read attacks r'botnet|zombie|c2\s*server', # Botnet infrastructure # New deep patterns r'IP\(src\s*=.*\.Random\b|IP\(src.*rand', # Scapy IP spoofing r'TCP\(flags\s*=\s*["\']S["\']\)', # Scapy SYN craft r'Ether\(\).*sendp?\(', # Scapy layer 2 injection r'sr1?\(.*timeout\s*=\s*0', # Scapy no-wait send r'RandIP\(\)|RandShort\(\)', # Scapy random generators r'def\s+attack\s*\(', # Attack function definition r'def\s+flood\s*\(', # Flood function definition r'def\s+ddos\s*\(', # DDoS function definition r'def\s+syn_flood\s*\(', # SYN flood function r'def\s+udp_flood\s*\(', # UDP flood function r'def\s+http_flood\s*\(', # HTTP flood function r'class\s+\w*(DDoS|Flood|Attack|Stress)', # DDoS class definition r'aiohttp\.ClientSession.*while.*True', # Async HTTP flood r'httpx\.AsyncClient.*while', # HTTPX async flood r'requests\.Session\(\).*while.*True', # Requests lib flood r'urllib3?\..*while.*True.*request', # urllib flood r'selenium.*while.*True.*get\(', # Selenium-based L7 flood r'puppeteer.*while.*true.*goto', # Puppeteer L7 flood (JS) r'playwright.*while.*true.*goto', # Playwright L7 flood r'curl.*while.*true|wget.*while.*true', # Bash curl/wget flood r'nc\s+-.*while|ncat.*while', # Netcat flood r'exec\s+\d+<>/dev/tcp/', # Bash /dev/tcp flood r'dd\s+if=/dev/urandom.*\|.*nc\b', # Random data pipe to netcat r'yes\s*\|.*nc\b', # Yes pipe to netcat (flood) r'base64.*decode.*socket.*send', # Encoded payload + socket flood r'eval\(.*base64.*socket', # Eval base64 attack script r'fork\s*\(\)|os\.fork\(\)', # Fork bomb / process multiply r'while.*true.*fork|:;\s*\(\)\s*\{', # Fork bomb patterns ] # CPU/memory thresholds CPU_MINING_THRESHOLD = 70.0 # Lowered: throttled miners run at 50-80% CPU_MINING_DEFINITE = 90.0 # Almost certainly mining at this level MEMORY_ABUSE_THRESHOLD_GB = 12.0 # AI models typically need 8-16GB+ @staticmethod def _test_regex_pattern(pattern): """Test if a regex pattern compiles successfully. Returns: - tuple: (True, compiled_pattern) if valid - tuple: (False, error_message) if invalid """ try: compiled = re.compile(pattern) return (True, compiled) except Exception as e: return (False, str(e)) @staticmethod def _safe_regex_search(pattern, text, flags=0): """Safely search with a regex pattern, returning False if pattern is invalid.""" try: return bool(re.search(pattern, text, flags)) except Exception: # Pattern is malformed - skip this check return False @classmethod def scan_container(cls, uuid, container_name): """Scan a running container for fraud indicators. Returns dict with: - ok: bool - threats: list of detected threats - risk_level: 'clean' | 'suspicious' | 'critical' Scan order is optimized: file scan first (instant, no docker needed), then docker-based scans. Early exit if critical threats found from files. """ threats = [] # -- Phase 1: File scan (instant - no docker commands needed) -- # This is the most reliable detection method: pure filesystem I/O. file_threats = cls._scan_files(uuid) threats.extend(file_threats) # Early exit: if file scan found critical threats, skip slow docker scans if any(t.get("severity") == "critical" for t in threats): logger.info(f"[{uuid}] File scan found {len(threats)} threat(s) - skipping docker scans") return { "ok": True, "threats": threats, "risk_level": "critical", "scanned_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } # -- Phase 2: Docker-based scans (slower, require docker exec) -- # 2a. Scan running processes (ps aux) proc_threats = cls._scan_processes(container_name) threats.extend(proc_threats) # Early exit check if any(t.get("severity") == "critical" for t in proc_threats): logger.info(f"[{uuid}] Process scan found critical threats - skipping remaining scans") return { "ok": True, "threats": threats, "risk_level": "critical", "scanned_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } # 2b. Advanced: scan /proc cmdlines for hidden miners cmdline_threats = cls._scan_proc_cmdlines(container_name) threats.extend(cmdline_threats) # 2c. Check CPU usage pattern (sustained high CPU = likely mining) cpu_threats = cls._check_cpu_pattern(container_name) threats.extend(cpu_threats) # 2d. Check memory usage (AI models consume huge amounts) mem_threats = cls._check_memory_abuse(container_name) threats.extend(mem_threats) # 2e. Check network connections for mining pools / C2 net_threats = cls._scan_network(container_name) threats.extend(net_threats) # 2f. Detect AI/ML workloads (pip packages, model files, GPU access) ai_threats = cls._detect_ai_workloads(uuid, container_name) threats.extend(ai_threats) # 2g. Advanced miner detection: check /proc for renamed/hidden miners hidden_threats = cls._detect_hidden_miners(container_name) threats.extend(hidden_threats) # 2h. Advanced DDoS attack detection (connection floods, raw sockets, bandwidth) ddos_threats = cls._detect_ddos_attack(uuid, container_name) threats.extend(ddos_threats) # -- Phase 3: Deep miner detection (anti-obfuscation, anti-wrapper) -- # 3a. Binary string analysis: read ELF binary content for miner signatures bin_threats = cls._scan_binary_strings(uuid) threats.extend(bin_threats) # 3b. Wrapper detection: bash/python/node wrapping a hidden miner wrapper_threats = cls._detect_wrapper_miners(uuid, container_name) threats.extend(wrapper_threats) # 3c. Fileless miner detection: memfd_create, /dev/shm, /proc execution fileless_threats = cls._detect_fileless_miners(container_name) threats.extend(fileless_threats) # 3d. Persistence detection: cron, at, systemd, .bashrc, .profile persist_threats = cls._detect_miner_persistence(uuid, container_name) threats.extend(persist_threats) # 3e. CPU-to-process correlation: high CPU + unknown binary = miner corr_threats = cls._correlate_cpu_with_processes(container_name) threats.extend(corr_threats) # 3f. Encoded config scanning: base64/hex pool URLs, wallet addresses encoded_threats = cls._scan_encoded_configs(uuid) threats.extend(encoded_threats) # 3g. Deep network: DNS queries to mining pools, env vars with pool URLs deep_net_threats = cls._deep_network_inspection(container_name) threats.extend(deep_net_threats) # -- Phase 4: Deep DDoS detection (anti-obfuscation, anti-wrapper) -- # 4a. DDoS binary string analysis: scan ELF binaries for attack tool signatures ddos_bin_threats = cls._scan_ddos_binary_strings(uuid) threats.extend(ddos_bin_threats) # 4b. DDoS wrapper detection: bash/python/node wrapping attack tools ddos_wrapper_threats = cls._detect_ddos_wrappers(uuid, container_name) threats.extend(ddos_wrapper_threats) # 4c. Botnet / C2 detection: IRC bots, Mirai variants, C2 beacons botnet_threats = cls._detect_botnet_c2(container_name) threats.extend(botnet_threats) # 4d. Network flood behavior analysis: packet rates, connection rates flood_threats = cls._analyze_network_flood_behavior(container_name) threats.extend(flood_threats) # 4e. DDoS persistence detection: cron-based attacks, respawn loops ddos_persist_threats = cls._detect_ddos_persistence(uuid, container_name) threats.extend(ddos_persist_threats) # 4f. Encoded DDoS payload scanning: base64/hex attack scripts ddos_encoded_threats = cls._scan_encoded_ddos_payloads(uuid) threats.extend(ddos_encoded_threats) # Determine risk level if any(t.get("severity") == "critical" for t in threats): risk_level = "critical" elif threats: risk_level = "suspicious" else: risk_level = "clean" return { "ok": True, "threats": threats, "risk_level": risk_level, "scanned_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } @classmethod def _scan_processes(cls, container_name): """Check running processes inside the container for known malware.""" threats = [] rc, out, _ = run_cmd( f"docker exec {container_name} ps aux 2>/dev/null || docker top {container_name} 2>/dev/null", timeout=10, ) if rc != 0 or not out: return threats for pattern, description in cls.MALICIOUS_PROCESS_PATTERNS: if re.search(pattern, out, re.IGNORECASE): severity = "critical" # AI workloads are critical fraud if "AI " in description: severity = "critical" threats.append({ "type": "malicious_process", "description": description, "severity": severity, "evidence": re.search(pattern, out, re.IGNORECASE).group()[:100], }) return threats @classmethod def _scan_proc_cmdlines(cls, container_name): """Read /proc/*/cmdline inside the container for hidden processes. Miners often rename their process (prctl PR_SET_NAME) to look like legitimate software. Reading /proc/*/cmdline reveals the real binary. """ threats = [] rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/cmdline; do echo \"==$p==\"; cat \"$p\" 2>/dev/null | tr \"\\0\" \" \"; echo; done'", timeout=15, ) if rc != 0 or not out: return threats miner_cmdline_patterns = [ (r'xmrig|cpuminer|minerd|ethminer|phoenixminer|nbminer|lolminer|gminer|ccminer' r'|srbminer|teamredminer|wildrig|nanominer|cgminer|bfgminer|claymore|ewbf' r'|xmr-stak|bzminer|rigel|onezerominer|cpuminer-opt|cpuminer-multi|tonminer', "Hidden crypto miner (process renamed)"), (r'stratum\+tcp|stratum\+ssl|stratum2\+tcp|stratum\+tls', "Hidden mining pool connection in cmdline"), (r'--algo\s*(cn|rx|gr|kawpow|ethash|randomx|cryptonight|ghostrider|etchash|progpow)', "Hidden mining algorithm in cmdline"), (r'--donate-level|--nicehash|--coin\s', "Hidden miner flags in cmdline"), (r'4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}', "Monero wallet address in cmdline"), (r'0x[0-9a-fA-F]{40}', "Ethereum wallet address in cmdline"), (r'--max-cpu-usage|--cpu-priority|--threads\s+\d+.*--algo', "CPU miner tuning flags"), (r'ollama|llama[\._-]?cpp|llama[\._-]?server|vllm|koboldcpp|localai|tabbyapi' r'|text-generation|comfyui|stable.diffusion|automatic1111|invokeai|gpt4all', "Hidden AI workload (process renamed)"), ] for line in out.splitlines(): if line.startswith("==") or not line.strip(): continue for pattern, description in miner_cmdline_patterns: if re.search(pattern, line, re.IGNORECASE): threats.append({ "type": "hidden_malicious_process", "description": description, "severity": "critical", "evidence": line.strip()[:200], }) return threats @classmethod def _scan_files(cls, uuid): """Scan server data directory recursively for suspicious and AI model files.""" threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): logger.info(f"[{uuid}] File scan: data dir does not exist: {data_dir}") return threats logger.info(f"[{uuid}] Starting file scan in {data_dir}") try: checked = 0 max_files = 2000 total_model_size = 0 # Walk recursively through the entire data directory (max depth 5) for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() # Don't go deeper continue # Skip known safe directories dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps"): dirs.clear() continue for filename in files: if checked >= max_files: break checked += 1 try: entry = Path(root) / filename name_lower = filename.lower() suffix = entry.suffix.lower() # Check filename against malicious patterns for bad_name in cls.MALICIOUS_FILE_PATTERNS: if bad_name in name_lower and bad_name != "config.json": threats.append({ "type": "malicious_file", "description": f"Suspicious file: {filename}", "severity": "critical" if any( m in name_lower for m in ["xmrig", "cpuminer", "minerd", "loic", ".gguf", ".safetensors", "ddos", "flood", "slowloris", "synflood", "udpflood", "icmpflood", "mirai", "gafgyt", "bashlite", "stresser", "booter", "goldeneye", "hulk", "xerxes", "torshammer", "hoic", "mhddos", "db1000n", "ufonet", "ddosify", "botnet", "kaiten", "tsunami", "hajime", "satori", "billgates", "xor_ddos", "sockstress", "amplifier", "reflector", "attack.py", "attack.sh", "dnscat", "iodine", "http_bomber", "http_killer", "cc_attack", "spoofer"] ) else "suspicious", "path": str(entry.relative_to(data_dir)), }) # Content-based scan for script files (detect DDoS/attack code inside) if suffix in (".py", ".sh", ".bash", ".pl", ".rb") and entry.stat().st_size < 512000: try: content = entry.read_text(errors="ignore")[:16384] content_lower = content.lower() ddos_indicators = 0 ddos_details = [] ddos_checks = [ ("sock_raw", "SOCK_RAW|socket.SOCK_RAW|AF_PACKET|IPPROTO_RAW"), ("sendto flood", r"sendto.*\bfor\b|\bwhile\b.*sendto|\.send\(.*\*\s*\d{3,}"), ("hping/flood", "hping|--flood|--syn.*flood|--rand-source"), ("threading flood", r"threading.*flood|thread.*attack|thread.*send|Thread\(target.*flood"), ("amplification", "amplif|spoof|reflect|dns.*amp|ntp.*amp|memcach.*amp"), ("iptables flush", "iptables.*-F|iptables.*flush|nft.*flush|ufw.*disable"), ("slowloris", "slowloris|slow.*header|keep.*alive.*attack|slow.*read|slow.*body"), ("DDoS import", r"import.*scapy|from.*scapy|import.*hping|from.*hping"), ("socket flood", r"socket\.socket.*for.*in.*range.*send|socket\(.*SOCK_DGRAM.*sendto"), ("HTTP flood", r"requests?\.(get|post).*while|urllib.*while.*true|aiohttp.*flood|httpx.*flood"), ("IP spoofing", r"IP\(src\s*=|IP\(dst\s*=.*random|fake.*ip|random.*source.*ip"), ("SYN flag craft", r"TCP\(flags\s*=.*S|TCP\(dport.*flags|SYN.*packet|syn_packet"), ("UDP payload", r"UDP\(dport.*send|udp.*payload.*random|random.*bytes.*send"), ("ICMP craft", r"ICMP\(|icmp.*flood|ping.*flood|icmp.*send"), ("multiprocessing", r"multiprocessing.*flood|Pool\(\d+\).*flood|ProcessPoolExecutor"), ("asyncio flood", r"asyncio.*flood|async.*flood|aiohttp.*session.*get.*while"), ("subprocess attack", r"subprocess.*(hping|nping|masscan|zmap|nmap|stress)"), ("target URL/IP", r"target\s*=\s*['\"]https?://|target_ip|target_host|victim"), ("attack func", r"def\s+(attack|flood|ddos|stress|bomb|blast|nuke|destroy)"), ("infinite loop", r"while\s+(True|1)\s*:.*socket|while\s+(True|1)\s*:.*send"), ("mass connect", r"for.*in.*range\(\d{3,}\).*connect|for.*range\(\d{3,}\).*socket"), ("packet forge", r"struct\.pack.*socket|pack\(.*\\x.*send|raw.*packet.*send"), ("botnet beacon", r"irc.*join|irc.*privmsg.*flood|c2.*connect|beacon.*send"), ("DNS tunnel", r"dns.*tunnel|dnscat|iodine.*tunnel|dns.*exfil"), ] for label, pattern in ddos_checks: if cls._safe_regex_search(pattern, content, re.IGNORECASE): ddos_indicators += 1 ddos_details.append(label) if ddos_indicators >= 2: threats.append({ "type": "ddos_attack_script", "description": f"DDoS attack script detected in {filename} ({', '.join(ddos_details[:4])})", "severity": "critical", "path": str(entry.relative_to(data_dir)), }) # Content-based miner detection in scripts content_lower = content.lower() miner_indicators = 0 miner_details = [] miner_checks = [ ("stratum URL", r"stratum\+tcp|stratum\+ssl|stratum2\+tcp"), ("pool domain", "|".join(re.escape(p) for p in cls.MINING_POOL_DOMAINS[:25])), ("miner binary", r"\bxmrig\b|\bcpuminer\b|\bminerd\b|\bethminer\b|\blolminer\b|\bgminer\b"), ("mining algo", r"\brandomx\b|\bcryptonight\b|\bethash\b|\bkawpow\b|\bghostrider\b"), ("miner config", r'"algo"\s*:|"pool"\s*:|"wallet"\s*:|"donate-level"'), ("wallet addr", r"4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}|0x[0-9a-fA-F]{40}"), ("hashrate ref", r"\bhashrate\b|\bh/s\b|\bkh/s\b|\bmh/s\b"), ("miner flag", r"--donate-level|--algo\s|--coin\s|--url\s.*:\d{4}|--user\s.*\.\w+"), ("download miner", r"(curl|wget)\s+[^\n]*(xmrig|miner|pool|stratum)"), ] for label, pattern in miner_checks: if cls._safe_regex_search(pattern, content, re.IGNORECASE): miner_indicators += 1 miner_details.append(label) if miner_indicators >= 2: threats.append({ "type": "miner_script_content", "description": (f"Crypto miner script detected in {filename} " f"({', '.join(miner_details[:5])})"), "severity": "critical", "path": str(entry.relative_to(data_dir)), }) elif miner_indicators == 1: threats.append({ "type": "suspicious_miner_reference", "description": f"Mining reference in script {filename} ({miner_details[0]})", "severity": "suspicious", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass # Detect AI model files by extension if suffix in cls.AI_MODEL_EXTENSIONS: try: fsize = entry.stat().st_size total_model_size += fsize if fsize > cls.AI_MODEL_SIZE_THRESHOLD: threats.append({ "type": "ai_model_file", "description": f"AI model file detected: {filename} ({fsize / (1024*1024):.0f}MB)", "severity": "critical", "path": str(entry.relative_to(data_dir)), }) else: threats.append({ "type": "ai_model_file", "description": f"AI model file: {filename} ({fsize / (1024*1024):.0f}MB)", "severity": "suspicious", "path": str(entry.relative_to(data_dir)), }) except OSError: pass # Check ELF binaries that shouldn't be there if not entry.suffix and os.access(str(entry), os.X_OK): try: with open(entry, "rb") as f: magic = f.read(4) if magic == b'\x7fELF': known_ok = ["java", "node", "python", "python3", "bash", "sh", "srcds_linux", "srcds_run", "hlds_linux", "bedrock_server", "PalServer", "FactoryServer", "7DaysToDieServer", "DayZServer", "Unturned", "ProjectZomboid"] if not any(ok.lower() in name_lower for ok in known_ok): threats.append({ "type": "unknown_binary", "description": f"Unknown executable binary: {filename}", "severity": "suspicious", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass # Heuristic: check JSON/YAML/CONF files for miner configs if name_lower in ("config.json", "pool.json", "pools.json", "miner.json", "settings.json") or \ suffix in (".json", ".yaml", ".yml", ".conf", ".cfg", ".toml"): try: content = entry.read_text(errors="ignore")[:8192] miner_hits = sum( 1 for p in cls.MINER_CONFIG_PATTERNS if cls._safe_regex_search(p, content, re.IGNORECASE) ) threshold = 3 if name_lower == "config.json" else 4 if miner_hits >= threshold: threats.append({ "type": "miner_config", "description": f"Crypto miner configuration file ({miner_hits} indicators)", "severity": "critical", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass except Exception as file_err: logger.debug(f"[{uuid}] Error scanning file {filename}: {file_err}") if checked >= max_files: break # Check for AI model directories ai_dirs = [".ollama", "huggingface", ".cache/huggingface", "models/ollama", "comfyui", "stable-diffusion"] for ai_dir in ai_dirs: d = data_dir / ai_dir if d.exists() and d.is_dir(): threats.append({ "type": "ai_model_directory", "description": f"AI/ML directory detected: {ai_dir}/", "severity": "critical", "path": ai_dir, }) # Summary: if total model files exceed 1GB, that's very suspicious if total_model_size > 1024 * 1024 * 1024: threats.append({ "type": "ai_model_storage", "description": f"Total AI model storage: {total_model_size / (1024**3):.1f}GB - AI hosting forbidden", "severity": "critical", }) except Exception as e: logger.warning(f"[{uuid}] Fraud file scan error: {e}", exc_info=True) if threats: logger.warning(f"[{uuid}] File scan found {len(threats)} threat(s): " f"{[t['description'][:60] for t in threats[:5]]}") else: logger.info(f"[{uuid}] File scan complete - no threats found (checked {checked} files)") return threats @classmethod def _check_cpu_pattern(cls, container_name): """Check if CPU usage is sustained at mining-level. Takes 5 samples over 10 seconds for reliable detection. Crypto miners typically peg all cores at 100% with very low variance. Throttled miners run at 50-80% with similarly low variance. """ threats = [] samples = [] for _ in range(5): rc, out, _ = run_cmd( f"docker stats --no-stream --format '{{{{.CPUPerc}}}}' {container_name}", timeout=5, ) if rc == 0 and out: try: cpu = float(out.strip().strip("'").replace("%", "")) samples.append(cpu) except ValueError: pass time.sleep(2) if len(samples) >= 3: avg_cpu = sum(samples) / len(samples) min_cpu = min(samples) max_cpu = max(samples) variance = max_cpu - min_cpu if avg_cpu >= cls.CPU_MINING_DEFINITE: # Very high CPU = almost certainly mining if variance < 10: threats.append({ "type": "crypto_mining_cpu_pattern", "description": (f"CPU mining signature: sustained {avg_cpu:.1f}% with " f"low variance ({variance:.1f}%) - almost certainly crypto mining"), "severity": "critical", "evidence": f"Samples: {[f'{s:.1f}%' for s in samples]}, avg={avg_cpu:.1f}%, var={variance:.1f}%", }) else: threats.append({ "type": "high_cpu_sustained", "description": f"Sustained high CPU usage ({avg_cpu:.1f}%) - possible crypto mining or AI inference", "severity": "suspicious", "evidence": f"Samples: {[f'{s:.1f}%' for s in samples]}", }) elif avg_cpu >= cls.CPU_MINING_THRESHOLD: # Moderate-high CPU with very low variance = throttled miner if variance < 8: threats.append({ "type": "throttled_miner_cpu_pattern", "description": (f"Throttled miner CPU signature: sustained {avg_cpu:.1f}% with " f"suspiciously low variance ({variance:.1f}%) - likely throttled miner"), "severity": "suspicious", "evidence": f"Samples: {[f'{s:.1f}%' for s in samples]}, avg={avg_cpu:.1f}%, var={variance:.1f}%", }) return threats @classmethod def _check_memory_abuse(cls, container_name): """Check for excessive memory usage (AI models typically need 8-16GB+).""" threats = [] rc, out, _ = run_cmd( f"docker stats --no-stream --format '{{{{.MemUsage}}}}' {container_name}", timeout=5, ) if rc != 0 or not out: return threats try: # Format: "1.5GiB / 4GiB" or "500MiB / 2GiB" usage_str = out.strip().strip("'").split("/")[0].strip() mem_gb = 0 if "GiB" in usage_str or "GB" in usage_str: mem_gb = float(re.sub(r'[^0-9.]', '', usage_str)) elif "MiB" in usage_str or "MB" in usage_str: mem_gb = float(re.sub(r'[^0-9.]', '', usage_str)) / 1024 if mem_gb >= cls.MEMORY_ABUSE_THRESHOLD_GB: threats.append({ "type": "excessive_memory", "description": f"Excessive memory usage ({mem_gb:.1f}GB) - possible AI model loaded in RAM", "severity": "suspicious", "evidence": out.strip(), }) except (ValueError, IndexError): pass return threats @classmethod def _scan_network(cls, container_name): """Check network connections for mining pools and C2 servers.""" threats = [] mining_ports = {"3333", "4444", "5555", "7777", "8888", "9999", "14444", "14433", "45560", "45700", "10128", "10256", "13333", "23333", "33333", "20535"} game_ports = {"7777", "8888", "27015", "25565", "30120"} suspicious_ports = mining_ports - game_ports # AI inference typical ports ai_ports = {"11434", # Ollama "8080", "5000", "7860", "3000", # Various AI UIs "5001", "8188", # ComfyUI } rc, out, _ = run_cmd( f"docker exec {container_name} sh -c 'ss -tnp 2>/dev/null || netstat -tnp 2>/dev/null' ", timeout=10, ) if rc != 0 or not out: return threats for line in out.splitlines(): line_lower = line.lower() if "estab" in line_lower or "established" in line_lower: # Check for known mining pool domains if any(kw in line_lower for kw in cls.MINING_POOL_DOMAINS): threats.append({ "type": "mining_pool_connection", "description": "Active connection to cryptocurrency mining pool", "severity": "critical", "evidence": line.strip()[:200], }) # Check for connections to suspicious mining ports for port in suspicious_ports: if f":{port}" in line: threats.append({ "type": "suspicious_port_connection", "description": f"Connection to common mining port :{port}", "severity": "suspicious", "evidence": line.strip()[:200], }) break # Check for AI inference servers LISTENING if "listen" in line_lower: for port in ai_ports: if f":{port}" in line: threats.append({ "type": "ai_service_listening", "description": f"AI inference service listening on port :{port}", "severity": "critical", "evidence": line.strip()[:200], }) break return threats @classmethod def _detect_ai_workloads(cls, uuid, container_name): """Detect AI/ML workloads: pip packages, model dirs, GPU access.""" threats = [] # Check for AI pip packages installed rc, out, _ = run_cmd( f"docker exec {container_name} sh -c 'pip list 2>/dev/null || pip3 list 2>/dev/null'", timeout=10, ) if rc == 0 and out: out_lower = out.lower() found_ai_packages = [] for pkg in cls.AI_PYTHON_PACKAGES: if pkg.lower() in out_lower: found_ai_packages.append(pkg) if len(found_ai_packages) >= 2: threats.append({ "type": "ai_packages_installed", "description": f"AI/ML packages installed: {', '.join(found_ai_packages[:10])}", "severity": "critical", "evidence": f"Found {len(found_ai_packages)} AI packages", }) # Check for .ollama directory (Ollama model store) data_dir = Path(DATA_DIR) / "servers" / uuid / "data" for ai_dir in [".ollama", "models", "huggingface", ".cache/huggingface", ".cache/torch", ".cache/lm-studio", "stable-diffusion-webui"]: check = data_dir / ai_dir if check.exists() and check.is_dir(): try: # Check if directory is non-trivial (has files) file_count = sum(1 for _ in check.iterdir()) if file_count > 0: threats.append({ "type": "ai_model_directory", "description": f"AI model directory found: {ai_dir}/ ({file_count} items)", "severity": "critical", }) except (PermissionError, OSError): pass # Check if GPU/CUDA is accessible inside container rc, out, _ = run_cmd( f"docker exec {container_name} sh -c 'nvidia-smi 2>/dev/null && echo GPU_FOUND'", timeout=5, ) if rc == 0 and "GPU_FOUND" in (out or ""): threats.append({ "type": "gpu_access", "description": "GPU access detected inside container - possible AI/mining workload", "severity": "critical", "evidence": "nvidia-smi accessible", }) return threats @classmethod def _detect_hidden_miners(cls, container_name): """Advanced miner detection: check for process masquerading and LD_PRELOAD tricks. Techniques detected: - Process name changed via prctl(PR_SET_NAME) to look like [kworker] or bash - LD_PRELOAD library injection to hide processes from ps - Deleted binaries still running (shows as '(deleted)' in /proc/exe) - Processes with suspicious high CPU but innocent names """ threats = [] # Check for deleted binaries still running (common miner trick: download, run, delete) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ls -la /proc/[0-9]*/exe 2>/dev/null | grep deleted'", timeout=10, ) if rc == 0 and out and "deleted" in out: for line in out.splitlines(): if "deleted" in line.lower(): threats.append({ "type": "deleted_binary_running", "description": "Running process with deleted binary - classic miner concealment", "severity": "critical", "evidence": line.strip()[:200], }) # Check for LD_PRELOAD in container environment (used to hook ps/top to hide processes) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/[0-9]*/environ 2>/dev/null | tr \"\\0\" \"\\n\" | grep -i LD_PRELOAD'", timeout=10, ) if rc == 0 and out and "LD_PRELOAD" in out: threats.append({ "type": "ld_preload_injection", "description": "LD_PRELOAD detected - possible process hiding rootkit", "severity": "critical", "evidence": out.strip()[:200], }) # Check for processes masquerading as kernel threads [kworker/...] rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ps aux 2>/dev/null | grep -E \"\\[k(worker|softirq|thread)\" | grep -v grep'", timeout=5, ) if rc == 0 and out and out.strip(): # In containers, there should never be real kernel threads threats.append({ "type": "kernel_thread_masquerade", "description": "Process masquerading as kernel thread - likely hidden miner", "severity": "critical", "evidence": out.strip()[:200], }) return threats # -- DDoS / Network Attack Thresholds (HARDENED) ---------------- # Max outbound established connections (game servers rarely need >30) DDOS_MAX_OUTBOUND_CONNECTIONS = 60 # Max outbound connections to a SINGLE IP (flood pattern) DDOS_MAX_CONNECTIONS_PER_IP = 15 # Max SYN_SENT connections (SYN flood outbound indicator) DDOS_MAX_SYN_SENT = 10 # Outbound bandwidth spike threshold (MB/s) - game servers send <10MB/s normally DDOS_BANDWIDTH_THRESHOLD_MBPS = 30 # Max total socket count (including TIME_WAIT etc.) DDOS_MAX_TOTAL_SOCKETS = 300 # Max UDP outbound connections (amplification attacks use UDP) DDOS_MAX_UDP_OUTBOUND = 30 @classmethod def _detect_ddos_attack(cls, uuid, container_name): """Advanced DDoS attack detection. Detects both outbound DDoS attacks (server used as attack source) and tools/scripts prepared for DDoS. Checks: 1. Connection flood analysis - excessive outbound TCP/UDP connections 2. SYN flood detection - many SYN_SENT connections (outbound SYN flood) 3. Connection fan-out - many connections to different IPs (botnet behavior) 4. Single-target flood - many connections to one IP (targeted DDoS) 5. RAW socket usage - needed for IP spoofing, SYN floods, ICMP floods 6. Outbound bandwidth analysis - TX bytes spike detection 7. UDP flood detection - excessive outbound UDP (amplification/reflection) 8. Amplification attack vectors - DNS(53), NTP(123), memcached(11211), SSDP(1900) 9. ICMP flood detection - excessive ping/ICMP traffic 10. DDoS scripts in files - Python/bash scripts with flood patterns 11. Iptables/nftables manipulation - attempts to disable firewall 12. Process argument analysis - flood flags in running commands """ threats = [] # -- 1. Connection flood analysis (TCP) --------------------- rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -tn 2>/dev/null || netstat -tn 2>/dev/null'", timeout=10, ) if rc == 0 and out: lines = [l for l in out.splitlines() if l.strip() and not l.startswith("State")] established = [l for l in lines if "ESTAB" in l.upper() or "ESTABLISHED" in l.upper()] syn_sent = [l for l in lines if "SYN-SENT" in l.upper() or "SYN_SENT" in l.upper()] time_wait = [l for l in lines if "TIME-WAIT" in l.upper() or "TIME_WAIT" in l.upper()] total_sockets = len(lines) # Count outbound connections (source is local, destination is remote) outbound_ips = {} for line in established: parts = line.split() # ss format: State Recv-Q Send-Q Local:Port Peer:Port # netstat format: tcp 0 0 Local:Port Foreign:Port State for part in parts: if ":" in part and not part.startswith("0.0.0.0") and not part.startswith(":::"): # Extract IP from ip:port ip = part.rsplit(":", 1)[0].strip("[]") if ip and not ip.startswith("127.") and not ip.startswith("0."): outbound_ips[ip] = outbound_ips.get(ip, 0) + 1 total_outbound = sum(outbound_ips.values()) # -- Total outbound connection flood ---- if total_outbound > cls.DDOS_MAX_OUTBOUND_CONNECTIONS: threats.append({ "type": "ddos_connection_flood", "description": (f"Excessive outbound connections: {total_outbound} " f"(threshold: {cls.DDOS_MAX_OUTBOUND_CONNECTIONS}) - " f"likely DDoS attack in progress"), "severity": "critical", "evidence": f"Top targets: {sorted(outbound_ips.items(), key=lambda x: -x[1])[:5]}", }) # -- Single-target flood (many connections to one IP) ---- for ip, count in outbound_ips.items(): if count > cls.DDOS_MAX_CONNECTIONS_PER_IP: threats.append({ "type": "ddos_single_target_flood", "description": (f"Connection flood to single target {ip}: {count} connections " f"- targeted DDoS attack"), "severity": "critical", "evidence": f"Target: {ip}, connections: {count}", }) # -- Connection fan-out (many different IPs = scanning or botnet) ---- if len(outbound_ips) > 50: threats.append({ "type": "ddos_connection_fanout", "description": (f"Connections to {len(outbound_ips)} different IPs - " f"possible port scanning or distributed attack"), "severity": "suspicious", "evidence": f"Unique target IPs: {len(outbound_ips)}", }) # -- SYN flood outbound ---- if len(syn_sent) > cls.DDOS_MAX_SYN_SENT: threats.append({ "type": "ddos_syn_flood_outbound", "description": (f"SYN flood detected: {len(syn_sent)} SYN_SENT connections - " f"outbound SYN flood attack"), "severity": "critical", "evidence": f"SYN_SENT count: {len(syn_sent)}, sample: {syn_sent[:3]}", }) # -- Socket exhaustion ---- if total_sockets > cls.DDOS_MAX_TOTAL_SOCKETS: threats.append({ "type": "ddos_socket_exhaustion", "description": (f"Socket exhaustion: {total_sockets} total sockets " f"(TIME_WAIT: {len(time_wait)}) - possible flood aftermath"), "severity": "suspicious", "evidence": f"Total: {total_sockets}, ESTABLISHED: {len(established)}, " f"SYN_SENT: {len(syn_sent)}, TIME_WAIT: {len(time_wait)}", }) # -- 2. UDP outbound analysis (amplification attacks) -------- rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -unp 2>/dev/null || netstat -unp 2>/dev/null'", timeout=10, ) if rc == 0 and out: udp_lines = [l for l in out.splitlines() if l.strip() and "udp" in l.lower()] # Check for amplification attack vectors (expanded) amp_ports = { "53": "DNS", "123": "NTP", "161": "SNMP", "389": "CLDAP", "1900": "SSDP", "11211": "Memcached", "19": "Chargen", "520": "RIPv1", "1434": "MSSQL", "17": "QOTD", "27960": "Quake", "5353": "mDNS", "111": "Portmap", "1604": "Citrix", "3283": "Apple-RDP", "3702": "WS-Discovery", "5683": "CoAP", "10001": "Ubiquiti", "137": "NetBIOS", "500": "IKE/ISAKMP", "623": "IPMI", "1194": "OpenVPN", "3389": "RDP", "5060": "SIP", "32414": "Plex/PMSSDP", "37810": "Dahua-DVR", "4730": "Gearman", "6881": "BitTorrent", } amp_hits = {} for line in udp_lines: for port, proto in amp_ports.items(): if f":{port}" in line: amp_hits[proto] = amp_hits.get(proto, 0) + 1 for proto, count in amp_hits.items(): if count >= 2: # Lowered threshold from 3 to 2 threats.append({ "type": "ddos_amplification_attack", "description": (f"{proto} amplification attack: {count} UDP connections " f"to {proto} reflectors - reflection/amplification DDoS"), "severity": "critical", "evidence": f"Protocol: {proto}, connections: {count}", }) # General UDP flood if len(udp_lines) > cls.DDOS_MAX_UDP_OUTBOUND: threats.append({ "type": "ddos_udp_flood", "description": (f"Excessive outbound UDP: {len(udp_lines)} connections - " f"possible UDP flood or amplification attack"), "severity": "critical", "evidence": f"UDP connections: {len(udp_lines)}", }) # -- 3. RAW socket detection (required for IP spoofing, SYN floods) -- rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -w 2>/dev/null; cat /proc/net/raw 2>/dev/null; cat /proc/net/raw6 2>/dev/null'", timeout=5, ) if rc == 0 and out: raw_lines = [l for l in out.splitlines() if l.strip() and not l.startswith("sl") and not l.startswith("State")] if len(raw_lines) > 0: threats.append({ "type": "ddos_raw_socket", "description": (f"RAW socket(s) detected ({len(raw_lines)}) - " f"used for IP spoofing, SYN floods, ICMP floods"), "severity": "critical", "evidence": "\n".join(raw_lines[:5])[:300], }) # -- 4. Outbound bandwidth analysis ------------------------- # Sample TX bytes twice with 3 second gap to calculate rate tx_samples = [] for _ in range(2): rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/net/dev 2>/dev/null'", timeout=5, ) if rc == 0 and out: for line in out.splitlines(): if "eth0" in line or "veth" in line: parts = line.split() try: # /proc/net/dev: iface bytes packets ... (RX) | bytes packets ... (TX) # TX bytes is the 10th field (index 9) if iface: is first if ":" in parts[0]: tx_bytes = int(parts[9]) else: tx_bytes = int(parts[10]) tx_samples.append(tx_bytes) except (ValueError, IndexError): pass break if len(tx_samples) < 2: time.sleep(3) if len(tx_samples) >= 2: tx_rate_bytes = (tx_samples[-1] - tx_samples[0]) / 3.0 # bytes per second tx_rate_mbps = tx_rate_bytes / (1024 * 1024) if tx_rate_mbps > cls.DDOS_BANDWIDTH_THRESHOLD_MBPS: threats.append({ "type": "ddos_bandwidth_abuse", "description": (f"Outbound bandwidth spike: {tx_rate_mbps:.1f} MB/s " f"(threshold: {cls.DDOS_BANDWIDTH_THRESHOLD_MBPS} MB/s) - " f"DDoS attack or data exfiltration"), "severity": "critical", "evidence": f"TX rate: {tx_rate_mbps:.1f} MB/s over 3s sample", }) # -- 5. ICMP flood detection -------------------------------- rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/net/snmp 2>/dev/null | grep -i icmp'", timeout=5, ) if rc == 0 and out: # Parse ICMP OutMsgs to check for ICMP flood icmp_lines = out.splitlines() for i, line in enumerate(icmp_lines): if line.startswith("Icmp:") and i + 1 < len(icmp_lines): values = icmp_lines[i + 1].split() try: # OutMsgs is typically at index 14 if len(values) > 14: out_msgs = int(values[14]) if out_msgs > 10000: threats.append({ "type": "ddos_icmp_flood", "description": (f"High ICMP output: {out_msgs} outbound messages - " f"possible ICMP/ping flood attack"), "severity": "suspicious", "evidence": f"ICMP OutMsgs: {out_msgs}", }) except (ValueError, IndexError): pass # -- 6. Firewall manipulation detection --------------------- rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'history 2>/dev/null; cat ~/.bash_history 2>/dev/null; " f"cat /root/.bash_history 2>/dev/null'", timeout=5, ) if rc == 0 and out: fw_patterns = [ (r'iptables\s+-(F|X|Z|P\s+\w+\s+ACCEPT)', "iptables firewall flush/disable"), (r'nft\s+flush\s+ruleset', "nftables firewall flush"), (r'ufw\s+disable', "UFW firewall disabled"), (r'systemctl\s+stop\s+(firewalld|iptables|nftables)', "Firewall service stopped"), (r'ip\s+route\s+add.*via', "IP route manipulation (traffic rerouting)"), (r'ip\s+addr\s+add', "IP address manipulation (spoofing preparation)"), (r'tc\s+qdisc', "Traffic control manipulation"), (r'sysctl\s+.*net\.ipv4\.ip_forward\s*=\s*1', "IP forwarding enabled (traffic relay)"), (r'sysctl\s+.*net\.ipv4\.conf\..*\.rp_filter\s*=\s*0', "Reverse path filter disabled (spoofing)"), (r'echo\s+1\s*>\s*/proc/sys/net/ipv4/ip_forward', "IP forwarding via proc (relay)"), (r'echo\s+0\s*>\s*/proc/sys/net/ipv4/conf/.*rp_filter', "RP filter disabled via proc"), (r'iptables\s+.*MASQUERADE', "iptables NAT masquerade (traffic proxy)"), (r'iptables\s+.*DNAT|iptables\s+.*REDIRECT', "iptables port redirect (traffic relay)"), (r'ip6?tables\s+-I\s+OUTPUT\s+-j\s+ACCEPT', "Allowing all outbound traffic"), ] for pattern, description in fw_patterns: if re.search(pattern, out, re.IGNORECASE): threats.append({ "type": "ddos_firewall_manipulation", "description": f"Firewall/network manipulation: {description}", "severity": "critical", "evidence": re.search(pattern, out, re.IGNORECASE).group()[:100], }) # -- 7. DDoS scripts / tools in data directory -------------- data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if data_dir.exists(): # Use comprehensive class-level DDOS_SCRIPT_PATTERNS ddos_script_patterns = cls.DDOS_SCRIPT_PATTERNS try: checked = 0 script_exts = {".py", ".sh", ".bash", ".pl", ".rb", ".js", ".php", ".go", ".c", ".cpp"} for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps"): dirs.clear() continue for filename in files: if checked > 500: break checked += 1 entry = Path(root) / filename suffix = entry.suffix.lower() if suffix not in script_exts: continue try: content = entry.read_text(errors="ignore")[:16384] ddos_hits = [] for pattern in ddos_script_patterns: if cls._safe_regex_search(pattern, content, re.IGNORECASE): ddos_hits.append(pattern.split(".*")[0][:30]) if len(ddos_hits) >= 2: threats.append({ "type": "ddos_attack_script", "description": (f"DDoS attack script detected: {filename} " f"({len(ddos_hits)} attack patterns)"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Patterns: {ddos_hits[:5]}", }) elif len(ddos_hits) == 1: threats.append({ "type": "ddos_suspicious_script", "description": f"Suspicious network script: {filename}", "severity": "suspicious", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass if checked > 500: break except Exception as e: logger.debug(f"DDoS script scan error: {e}") # -- 8. Process argument flood flags ------------------------ rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/cmdline; do " f"cat \"$p\" 2>/dev/null | tr \"\\0\" \" \"; echo; done'", timeout=10, ) if rc == 0 and out: flood_cmdline_patterns = [ (r'--flood\b', "Flood flag in process arguments"), (r'--rand-source', "Random source IP (IP spoofing)"), (r'--rand-dest', "Random destination (scanning)"), (r'--(syn|ack|rst|fin|push|urg)\b', "TCP flag manipulation"), (r'--count\s+0\b', "Infinite packet count (endless flood)"), (r'-c\s+0\s', "Infinite count flag"), (r'--fast\b', "Fast mode (maximized sending rate)"), (r'--faster\b', "Faster mode (no delay)"), (r'-i\s*u?\d+\b.*send', "Microsecond interval sending (flood speed)"), (r'SOCK_RAW|IPPROTO_RAW|IPPROTO_ICMP', "Raw socket in process (packet crafting)"), (r'--attack[_-]?(method|type|mode)', "DDoS attack method specification"), (r'--(target|victim|host)\s+\S+:\d+', "DDoS target IP:port specification"), (r'--threads?\s+\d{2,}', "High thread count (DDoS indicator)"), (r'--duration\s+\d+', "Attack duration parameter"), (r'--packets?\s+\d{4,}', "High packet count parameter"), (r'--connections?\s+\d{3,}', "High connection count parameter"), (r'--rpc\s+\d{2,}', "High requests-per-connection (DDoS)"), (r'\bhping3?\b.*--flood', "hping flood mode"), (r'\bnping\b.*--rate\s+\d{3,}', "Nping high rate mode"), (r'\bmasscan\b.*--rate\s+\d{4,}', "Masscan high rate scanning"), (r'\bzmap\b.*--rate', "ZMap internet scanning"), (r'\bdd\b.*if=/dev/(zero|urandom).*\|\s*(nc|ncat)\b', "Data pipe to netcat (flood)"), (r'(slowloris|goldeneye|hulk|xerxes|torshammer|pyloris)', "Known DDoS tool execution"), (r'(mhddos|db1000n|ufonet|ddosify)', "Modern DDoS framework execution"), (r'(mirai|gafgyt|bashlite|kaiten|tsunami)', "Botnet execution"), (r'stress(-ng)?\s+--cpu\s+\d+.*--timeout', "CPU stress test"), (r'/dev/tcp/\S+/\d+', "Bash /dev/tcp connection (flood vector)"), ] for line in out.splitlines(): if not line.strip(): continue for pattern, description in flood_cmdline_patterns: if re.search(pattern, line, re.IGNORECASE): threats.append({ "type": "ddos_flood_process", "description": f"Active DDoS process: {description}", "severity": "critical", "evidence": line.strip()[:200], }) # -- 9. Packet rate analysis via /proc/net/snmp ------------- # Check TCP OutSegs (segments sent) - very high = flood rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/net/snmp 2>/dev/null | grep -i tcp'", timeout=5, ) if rc == 0 and out: tcp_lines = out.splitlines() for i, line in enumerate(tcp_lines): if line.startswith("Tcp:") and i + 1 < len(tcp_lines): values = tcp_lines[i + 1].split() try: # AttemptFails (index 7), OutSegs (index 10), RetransSegs (index 11) if len(values) > 11: attempt_fails = int(values[7]) out_segs = int(values[10]) retrans = int(values[11]) # Very high retransmission = sending to non-responsive targets (DDoS) if retrans > 10000 and (attempt_fails > 5000 or out_segs > 1000000): threats.append({ "type": "ddos_tcp_flood_stats", "description": (f"TCP flood signature in /proc/net/snmp: " f"OutSegs={out_segs}, RetransSegs={retrans}, " f"AttemptFails={attempt_fails}"), "severity": "critical", "evidence": f"OutSegs: {out_segs}, Retrans: {retrans}, Fails: {attempt_fails}", }) except (ValueError, IndexError): pass # -- 10. Conntrack table overflow check --------------------- rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null'", timeout=3, ) if rc == 0 and out: try: conntrack_count = int(out.strip()) if conntrack_count > 10000: threats.append({ "type": "ddos_conntrack_overflow", "description": (f"Conntrack table near overflow: {conntrack_count} entries - " f"connection flood in progress"), "severity": "critical" if conntrack_count > 50000 else "suspicious", "evidence": f"nf_conntrack_count: {conntrack_count}", }) except ValueError: pass return threats # -------------------------------------------------------------- # Phase 4: Deep DDoS Detection (Anti-Obfuscation / Anti-Wrapper) # -------------------------------------------------------------- @classmethod def _scan_ddos_binary_strings(cls, uuid): """Scan ELF binaries for DDoS tool signatures. Even if a DDoS tool is renamed to "bash", "java", "update-service", the compiled binary still contains strings like "flood_start", "SOCK_RAW", "amplification", "target_ip", "packets_sent", etc. Also detects Mirai/Gafgyt/Bashlite botnet signatures in binaries. """ threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): return threats try: checked_binaries = 0 max_binaries = 50 max_size = 10 * 1024 * 1024 # 10MB per binary for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps"): dirs.clear() continue for filename in files: if checked_binaries >= max_binaries: break entry = Path(root) / filename try: fsize = entry.stat().st_size if fsize < 10000 or fsize > max_size: continue # Check if ELF binary (starts with \x7fELF) with open(entry, "rb") as f: magic = f.read(4) if magic != b"\x7fELF": continue checked_binaries += 1 # Read in chunks and search for DDoS signatures f.seek(0) ddos_hits = [] chunk_size = 256 * 1024 read_total = 0 while read_total < max_size: chunk = f.read(chunk_size) if not chunk: break read_total += len(chunk) for sig in cls.DDOS_BINARY_STRINGS: if sig in chunk and sig.decode("utf-8", errors="ignore") not in [h[0] for h in ddos_hits]: ddos_hits.append((sig.decode("utf-8", errors="ignore"), "binary_string")) # Many DDoS signatures = definitely an attack tool if len(ddos_hits) >= 5: threats.append({ "type": "ddos_tool_binary", "description": (f"DDoS attack tool binary detected: {filename} " f"({len(ddos_hits)} attack signatures)"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Signatures: {[h[0] for h in ddos_hits[:8]]}", }) elif len(ddos_hits) >= 3: threats.append({ "type": "ddos_suspicious_binary", "description": (f"Suspicious binary with DDoS indicators: {filename} " f"({len(ddos_hits)} signatures)"), "severity": "suspicious", "path": str(entry.relative_to(data_dir)), "evidence": f"Signatures: {[h[0] for h in ddos_hits[:6]]}", }) except (OSError, PermissionError): pass if checked_binaries >= max_binaries: break except Exception as e: logger.debug(f"DDoS binary string scan error: {e}") return threats @classmethod def _detect_ddos_wrappers(cls, uuid, container_name): """Detect bash/python/node scripts that wrap or download DDoS tools. Catches patterns like: - wget/curl downloading known DDoS tools then executing - Python scripts importing socket + raw flood logic - Bash scripts running compiled DDoS binaries from hidden dirs - Base64-encoded attack payloads being decoded and executed - Process renaming via exec -a to disguise DDoS tools - LD_PRELOAD hiding DDoS processes from ps/top - nohup/setsid/disown to keep DDoS running after disconnect """ threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): return threats # Wrapper indicators in script files wrapper_indicators = [ (r'(curl|wget)\s+[^\n]*(ddos|flood|stress|booter|attack|mirai|gafgyt|loic|hoic|xerxes)', "DDoS tool download"), (r'(curl|wget)\s+[^\n]*\|\s*(bash|sh|python|perl|ruby)', "Download-and-execute attack script"), (r'chmod\s+\+?[0-7]*x[^\n]*(flood|attack|ddos|stress|bot)', "Making DDoS tool executable"), (r'exec\s+-a\s+["\'][^"\']*["\'].*\b(flood|attack|ddos|stress)', "Process renamed DDoS tool (exec -a)"), (r'LD_PRELOAD=.*\b(flood|attack|ddos)', "LD_PRELOAD hiding DDoS process"), (r'nohup\s+.*\b(flood|attack|ddos|stress|hping|sendto)', "DDoS tool running via nohup"), (r'setsid\s+.*\b(flood|attack|ddos|stress)', "DDoS tool running via setsid"), (r'disown\s+.*\b(flood|attack|ddos)', "DDoS tool disowned from shell"), (r'screen\s+-[dD]m\s+.*\b(flood|attack|ddos|stress)', "DDoS tool in detached screen"), (r'tmux.*\b(flood|attack|ddos|stress)', "DDoS tool in tmux session"), (r'/dev/shm/[^\s]*\b(flood|attack|ddos)', "DDoS tool running from /dev/shm"), (r'/tmp/\.[^\s]*\b(flood|attack|ddos)', "DDoS tool running from hidden /tmp dir"), (r'(python[23]?|perl|ruby|node)\s+-[ec]\s+.*socket.*send', "Inline attack script execution"), (r'(python[23]?|perl|ruby|node)\s+-[ec]\s+.*flood', "Inline flood script execution"), (r'base64\s+-d.*\|\s*(bash|sh|python)', "Base64-decoded attack script"), (r'echo\s+[A-Za-z0-9+/=]{40,}.*\|\s*base64\s+-d\s*\|\s*(bash|sh)', "Encoded attack payload execution"), (r'while\s+true\s*;\s*do\s.*\b(curl|wget|nc|ncat|hping)', "Infinite loop network flood in bash"), (r'for\s+\w+\s+in\s+\$\(seq\s+\d{3,}\).*\b(curl|nc|ncat)', "High-iteration network loop"), (r'xargs\s+-P\s+\d{2,}.*\b(curl|wget|nc)', "Parallel curl/wget flood (xargs)"), (r'parallel\s+.*\b(curl|wget|nc|ncat)', "GNU parallel network flood"), ] try: checked = 0 script_exts = {".py", ".sh", ".bash", ".pl", ".rb", ".js", ".php", ".go", ".c", ".cpp"} for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps"): dirs.clear() continue for filename in files: if checked > 500: break checked += 1 entry = Path(root) / filename suffix = entry.suffix.lower() if suffix not in script_exts: continue try: if entry.stat().st_size > 512000: continue content = entry.read_text(errors="ignore")[:32768] hits = [] for pattern, description in wrapper_indicators: if cls._safe_regex_search(pattern, content, re.IGNORECASE): hits.append(description) if len(hits) >= 2: threats.append({ "type": "ddos_wrapper_script", "description": (f"DDoS wrapper/launcher detected: {filename} " f"({', '.join(hits[:4])})"), "severity": "critical", "path": str(entry.relative_to(data_dir)), }) elif len(hits) == 1: threats.append({ "type": "ddos_suspicious_wrapper", "description": f"Suspicious DDoS wrapper indicator in {filename}: {hits[0]}", "severity": "suspicious", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass if checked > 500: break except Exception as e: logger.debug(f"DDoS wrapper scan error: {e}") # Also check running processes for live wrapper patterns rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/cmdline; do " f"cat \"$p\" 2>/dev/null | tr \"\\0\" \" \"; echo; done'", timeout=10, ) if rc == 0 and out: live_wrapper_patterns = [ (r'(curl|wget)\s+[^\n]*\|\s*(bash|sh|python)', "Live download-execute DDoS"), (r'exec\s+-a\s+["\'][^"\']*["\']', "Live process rename (exec -a disguise)"), (r'base64\s+-d\s*\|\s*(bash|sh|python)', "Live base64-decoded attack execution"), (r'(python|perl|ruby|node)\s+-[ec]\s+.*socket.*send', "Live inline socket flood"), (r'while\s+true.*\b(hping|nping|nc\s+-u)', "Live infinite loop network tool"), (r'/dev/shm/\S+', "Process running from /dev/shm (fileless attack)"), (r'/tmp/\.\S+', "Process running from hidden /tmp directory"), (r'nohup\s+.*\b(flood|attack|hping|stress)', "Live nohup DDoS process"), ] for line in out.splitlines(): if not line.strip(): continue for pattern, description in live_wrapper_patterns: if re.search(pattern, line, re.IGNORECASE): threats.append({ "type": "ddos_live_wrapper", "description": f"Live DDoS wrapper: {description}", "severity": "critical", "evidence": line.strip()[:200], }) return threats @classmethod def _detect_botnet_c2(cls, container_name): """Detect botnet Command & Control (C2) communication. Catches: - IRC-based botnet controllers (Mirai, Kaiten, Gafgyt) - HTTP-based C2 beacons - DNS-based C2 tunneling - Connections to known booter/stresser/C2 domains - Process behavior typical of botnet zombies - Hardcoded C2 IPs/domains in process environment """ threats = [] # 1. Check for IRC connections (classic botnet C2 channel) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -tnp 2>/dev/null | grep -E \":(6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6697|7000|7070|8067)\"'", timeout=5, ) if rc == 0 and out and out.strip(): irc_lines = [l for l in out.splitlines() if l.strip()] if irc_lines: threats.append({ "type": "botnet_irc_c2", "description": (f"IRC connection detected ({len(irc_lines)} connections) - " f"classic botnet C2 channel (Mirai/Kaiten/Gafgyt)"), "severity": "critical", "evidence": "\n".join(irc_lines[:5])[:300], }) # 2. Check for connections to known booter/C2 domains rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /etc/hosts 2>/dev/null; cat /etc/resolv.conf 2>/dev/null'", timeout=5, ) if rc == 0 and out: for domain in cls.BOOTER_C2_DOMAINS: if domain.lower() in out.lower(): threats.append({ "type": "botnet_c2_domain", "description": f"Botnet/DDoS C2 domain reference: {domain}", "severity": "critical", "evidence": f"Found in hosts/resolv.conf: {domain}", }) # 3. Scan ALL process environment variables for C2/booter indicators rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/environ; do " f"cat \"$p\" 2>/dev/null | tr \"\\0\" \"\\n\"; done'", timeout=10, ) if rc == 0 and out: c2_env_patterns = [ (r'C2_SERVER|C2_HOST|C2_URL|CNC_SERVER|CNC_HOST', "C2 server env variable"), (r'BOT_SERVER|BOT_HOST|BOT_MASTER|BOT_CHANNEL', "Bot controller env variable"), (r'IRC_SERVER|IRC_CHANNEL|IRC_NICK|IRC_PORT', "IRC bot env variable"), (r'ATTACK_TARGET|ATTACK_METHOD|ATTACK_PORT|ATTACK_TIME', "Attack config env variable"), (r'FLOOD_TARGET|FLOOD_METHOD|FLOOD_PORT|FLOOD_TIME', "Flood config env variable"), (r'DDOS_TARGET|DDOS_METHOD|DDOS_PORT|DDOS_DURATION', "DDoS config env variable"), (r'TARGET_IP|TARGET_HOST|TARGET_PORT|TARGET_URL', "Attack target env variable"), (r'VICTIM_IP|VICTIM_HOST|VICTIM_URL', "Victim env variable"), (r'STRESSER_API|BOOTER_API|BOOTER_KEY|STRESS_API', "Booter API env variable"), (r'(stresser|booter|ddos|cnc|c2)\.(com|net|io|xyz|to|cc|ru)', "C2/booter domain in env"), ] for line in out.splitlines(): for pattern, description in c2_env_patterns: if re.search(pattern, line, re.IGNORECASE): threats.append({ "type": "botnet_c2_env", "description": f"Botnet C2 indicator in env: {description}", "severity": "critical", "evidence": line.strip()[:200], }) # 4. Check for botnet-like process behavior rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ps aux 2>/dev/null | head -100'", timeout=5, ) if rc == 0 and out: botnet_process_patterns = [ (r'\b(dvrHelper|sora|owari|josho|tsunami)\b', "Known botnet variant"), (r'\b(yakuza|sakura|reaper|fbot|echobot)\b', "Known IoT botnet"), (r'\./\w{1,4}\s*$', "Unnamed short binary execution (botnet pattern)"), (r'/tmp/\.\w+\s', "Hidden binary in /tmp (botnet dropper)"), (r'/var/tmp/\.\w+\s', "Hidden binary in /var/tmp (botnet dropper)"), (r'/dev/shm/\w+\s', "Binary in /dev/shm (fileless botnet)"), (r'busybox\s+(wget|tftp|curl)', "BusyBox download (Mirai-like dropper pattern)"), (r'tftp\s+-g\s', "TFTP download (classic Mirai infection)"), (r'\.\/[a-z]{1,8}\s*$', "Short unnamed binary from current dir (bot)"), ] for pattern, description in botnet_process_patterns: if re.search(pattern, out, re.IGNORECASE): match = re.search(pattern, out, re.IGNORECASE) threats.append({ "type": "botnet_process", "description": f"Botnet process detected: {description}", "severity": "critical", "evidence": match.group()[:200] if match else "", }) # 5. Check for high number of outbound connections to different IPs on same port # (botnet scatter pattern - attacking multiple targets or scanning) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -tn 2>/dev/null | grep -i estab'", timeout=5, ) if rc == 0 and out: port_ips = {} # port -> set of IPs for line in out.splitlines(): parts = line.split() for part in parts: if ":" in part: try: ip_port = part.rsplit(":", 1) if len(ip_port) == 2: port = ip_port[1] ip = ip_port[0].strip("[]") if port.isdigit() and not ip.startswith("127."): port_ips.setdefault(port, set()).add(ip) except (ValueError, IndexError): pass for port, ips in port_ips.items(): if len(ips) > 20 and port not in ("443", "80", "53"): threats.append({ "type": "botnet_scatter_attack", "description": (f"Botnet scatter pattern: {len(ips)} different IPs on port {port} " f"- distributed attack or port scanning"), "severity": "critical", "evidence": f"Port: {port}, unique IPs: {len(ips)}, sample: {list(ips)[:5]}", }) # 6. Check for Mirai-specific telnet scanning rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -tnp 2>/dev/null | grep -E \":23\\b\" | wc -l'", timeout=5, ) if rc == 0 and out: try: telnet_conns = int(out.strip()) if telnet_conns > 5: threats.append({ "type": "botnet_telnet_scan", "description": (f"Telnet scanning detected: {telnet_conns} connections to port 23 " f"- Mirai-like IoT scanning"), "severity": "critical", "evidence": f"Telnet connections: {telnet_conns}", }) except ValueError: pass return threats @classmethod def _analyze_network_flood_behavior(cls, container_name): """Advanced network behavior analysis to detect active flood attacks. Goes beyond simple connection counting - analyzes: 1. Packet rate per second (TX packets via /proc/net/dev) 2. Connection creation rate (new connections per second) 3. Port entropy (attacking many ports = port scan / random port flood) 4. Outbound traffic asymmetry (TX >> RX = attack sending data) 5. Socket state distribution (many SYN_SENT = SYN flood) 6. UDP datagram rate from /proc/net/snmp """ threats = [] # 1. Packet rate analysis (sample twice over 3 seconds) samples = [] for _ in range(2): rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/net/dev 2>/dev/null'", timeout=5, ) if rc == 0 and out: for line in out.splitlines(): if "eth0" in line or "veth" in line: parts = line.split() try: if ":" in parts[0]: # RX: bytes(1), packets(2) | TX: bytes(9), packets(10) rx_bytes = int(parts[1]) rx_pkts = int(parts[2]) tx_bytes = int(parts[9]) tx_pkts = int(parts[10]) else: rx_bytes = int(parts[2]) rx_pkts = int(parts[3]) tx_bytes = int(parts[10]) tx_pkts = int(parts[11]) samples.append({ "rx_bytes": rx_bytes, "rx_pkts": rx_pkts, "tx_bytes": tx_bytes, "tx_pkts": tx_pkts, "time": time.time(), }) except (ValueError, IndexError): pass break if len(samples) < 2: time.sleep(3) if len(samples) >= 2: dt = max(samples[-1]["time"] - samples[0]["time"], 0.1) tx_pps = (samples[-1]["tx_pkts"] - samples[0]["tx_pkts"]) / dt rx_pps = (samples[-1]["rx_pkts"] - samples[0]["rx_pkts"]) / dt tx_bps = (samples[-1]["tx_bytes"] - samples[0]["tx_bytes"]) / dt rx_bps = (samples[-1]["rx_bytes"] - samples[0]["rx_bytes"]) / dt # Very high TX packet rate = flood (game servers typically <1000 pps) if tx_pps > 10000: threats.append({ "type": "ddos_high_packet_rate", "description": (f"Extremely high TX packet rate: {tx_pps:.0f} pps " f"(normal game traffic: <1000 pps) - active flood attack"), "severity": "critical", "evidence": f"TX: {tx_pps:.0f} pps, RX: {rx_pps:.0f} pps", }) elif tx_pps > 5000: threats.append({ "type": "ddos_elevated_packet_rate", "description": f"Elevated TX packet rate: {tx_pps:.0f} pps - possible flood", "severity": "suspicious", "evidence": f"TX: {tx_pps:.0f} pps, RX: {rx_pps:.0f} pps", }) # Traffic asymmetry: TX >> RX means sending much more than receiving # DDoS tools send floods outbound with minimal response if tx_bps > 0 and rx_bps > 0: asymmetry = tx_bps / max(rx_bps, 1) if asymmetry > 20 and tx_bps > 1_000_000: # 20:1 ratio, >1MB/s outbound threats.append({ "type": "ddos_traffic_asymmetry", "description": (f"Extreme TX/RX asymmetry: {asymmetry:.0f}:1 " f"(TX: {tx_bps/1024/1024:.1f} MB/s) - outbound flood pattern"), "severity": "critical", "evidence": f"TX: {tx_bps:.0f} B/s, RX: {rx_bps:.0f} B/s, ratio: {asymmetry:.0f}:1", }) # Packet rate with tiny packets = SYN flood / UDP flood (many small packets) if tx_pps > 5000: avg_pkt_size = tx_bps / max(tx_pps, 1) if avg_pkt_size < 100: # Less than 100 bytes per packet threats.append({ "type": "ddos_small_packet_flood", "description": (f"Small packet flood: {tx_pps:.0f} pps, avg {avg_pkt_size:.0f} bytes " f"- SYN/UDP/ICMP flood pattern"), "severity": "critical", "evidence": f"PPS: {tx_pps:.0f}, avg size: {avg_pkt_size:.0f} bytes", }) # 2. Socket state analysis (advanced) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -s 2>/dev/null'", timeout=5, ) if rc == 0 and out: # Parse summary: TCP: X (estab Y, closed Z, orphaned W, timewait T) syn_sent_match = re.search(r'synSent\s*[:=]\s*(\d+)', out, re.IGNORECASE) timewait_match = re.search(r'timewait\s*[:=]\s*(\d+)', out, re.IGNORECASE) closed_match = re.search(r'closed\s*[:=]\s*(\d+)', out, re.IGNORECASE) if syn_sent_match: syn_sent = int(syn_sent_match.group(1)) if syn_sent > 50: threats.append({ "type": "ddos_syn_flood_advanced", "description": (f"Active SYN flood: {syn_sent} SYN_SENT sockets " f"- outbound SYN flood attack in progress"), "severity": "critical", "evidence": f"SYN_SENT: {syn_sent}", }) if timewait_match: timewait = int(timewait_match.group(1)) if timewait > 1000: threats.append({ "type": "ddos_timewait_flood", "description": (f"TIME_WAIT accumulation: {timewait} sockets " f"- recent connection flood aftermath"), "severity": "suspicious", "evidence": f"TIME_WAIT: {timewait}", }) # 3. UDP datagram rate from /proc/net/snmp rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/net/snmp 2>/dev/null | grep -i udp'", timeout=5, ) if rc == 0 and out: udp_lines = out.splitlines() for i, line in enumerate(udp_lines): if line.startswith("Udp:") and i + 1 < len(udp_lines): values = udp_lines[i + 1].split() try: if len(values) >= 4: out_datagrams = int(values[3]) # OutDatagrams if out_datagrams > 1000000: threats.append({ "type": "ddos_udp_flood_stats", "description": (f"Massive UDP output: {out_datagrams} datagrams sent " f"- UDP flood or amplification attack"), "severity": "critical", "evidence": f"UDP OutDatagrams: {out_datagrams}", }) except (ValueError, IndexError): pass # 4. Check for many CLOSE_WAIT / FIN_WAIT states (connection flood cleanup) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -tn state close-wait 2>/dev/null | wc -l; " f"ss -tn state fin-wait-1 2>/dev/null | wc -l; " f"ss -tn state fin-wait-2 2>/dev/null | wc -l'", timeout=5, ) if rc == 0 and out: try: counts = [int(x.strip()) for x in out.strip().splitlines() if x.strip().isdigit()] total_closing = sum(counts) if total_closing > 200: threats.append({ "type": "ddos_connection_flood_aftermath", "description": (f"Connection flood aftermath: {total_closing} closing sockets " f"(CLOSE_WAIT/FIN_WAIT) - recent or ongoing flood"), "severity": "suspicious", "evidence": f"Closing sockets: {total_closing}", }) except (ValueError, IndexError): pass return threats @classmethod def _detect_ddos_persistence(cls, uuid, container_name): """Detect DDoS attack persistence mechanisms. Catches: - Crontab entries scheduling DDoS attacks - .bashrc/.profile injection to restart attacks on login - Systemd services for DDoS tools - Respawn loops that restart attacks after they're killed - At jobs scheduling attacks - Init.d scripts for DDoS tools """ threats = [] # 1. Check crontab for DDoS-related entries cron_sources = [ "crontab -l 2>/dev/null", "cat /etc/crontab 2>/dev/null", "cat /var/spool/cron/crontabs/* 2>/dev/null", "cat /var/spool/cron/* 2>/dev/null", "ls /etc/cron.d/ 2>/dev/null && cat /etc/cron.d/* 2>/dev/null", ] cron_cmd = " ; ".join(cron_sources) rc, out, _ = run_cmd( f"docker exec {container_name} sh -c '{cron_cmd}'", timeout=10, ) if rc == 0 and out: ddos_cron_patterns = [ (r'(flood|attack|ddos|stress|boost|hping|nping|xerxes|loic|hoic|mirai|gafgyt)', "DDoS tool in cron"), (r'(curl|wget)\s+[^\n]*\|\s*(bash|sh|python)', "Download-execute in cron"), (r'/dev/shm/\S+', "Binary from /dev/shm in cron (fileless attack)"), (r'/tmp/\.\S+', "Hidden /tmp binary in cron"), (r'(python|perl|ruby|node)\s+-[ec]\s+.*socket', "Inline socket script in cron"), (r'nohup.*socket|nohup.*(flood|attack)', "Nohup DDoS in cron"), (r'\*/\d+\s+\*\s+\*\s+\*\s+\*.*\.(sh|py|pl)\b', "Frequent script execution in cron"), ] for line in out.splitlines(): if line.strip().startswith("#") or not line.strip(): continue for pattern, description in ddos_cron_patterns: if re.search(pattern, line, re.IGNORECASE): threats.append({ "type": "ddos_persistence_cron", "description": f"DDoS persistence in crontab: {description}", "severity": "critical", "evidence": line.strip()[:200], }) # 2. Check .bashrc / .profile for DDoS injection rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat ~/.bashrc /root/.bashrc /home/*/.bashrc " f"~/.profile /root/.profile /home/*/.profile " f"~/.bash_profile /root/.bash_profile /home/*/.bash_profile " f"2>/dev/null'", timeout=5, ) if rc == 0 and out: shell_ddos_patterns = [ (r'(flood|attack|ddos|stress|hping|xerxes|loic|mirai)', "DDoS tool in shell profile"), (r'nohup\s+.*\b(flood|attack|ddos|nc\s+-e|/dev/tcp)', "Nohup DDoS in profile"), (r'(curl|wget)\s+[^\n]*\|\s*(bash|sh)', "Download-execute in profile"), (r'/dev/shm/\S+', "Fileless binary in profile"), (r'while\s+true.*\b(flood|attack|hping|sendto)', "Attack respawn loop in profile"), ] for line in out.splitlines(): if line.strip().startswith("#") or not line.strip(): continue for pattern, description in shell_ddos_patterns: if re.search(pattern, line, re.IGNORECASE): threats.append({ "type": "ddos_persistence_profile", "description": f"DDoS persistence in shell profile: {description}", "severity": "critical", "evidence": line.strip()[:200], }) # 3. Check for respawn/watchdog scripts that restart DDoS tools data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if data_dir.exists(): try: checked = 0 for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 4: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam"): dirs.clear() continue for filename in files: if checked > 300: break checked += 1 entry = Path(root) / filename suffix = entry.suffix.lower() if suffix not in (".sh", ".bash", ".py", ".pl"): continue try: if entry.stat().st_size > 100000: continue content = entry.read_text(errors="ignore")[:8192] # Respawn pattern: while true + start attack + sleep respawn_patterns = [ (r'while\s+(true|1)\s*[;:]\s*do.*\b(flood|attack|ddos|hping|xerxes|stress)', "DDoS respawn loop"), (r'while\s+(true|1).*sleep\s+\d+.*\b(flood|attack|ddos)', "DDoS watchdog script"), (r'until\s+false.*\b(flood|attack|ddos)', "DDoS until-false respawn"), (r'trap\s+.*\b(flood|attack|ddos)', "DDoS trap restart handler"), (r'pgrep.*\b(flood|attack|ddos).*\|\|.*start', "DDoS process monitor/restart"), (r'if\s+!\s*pgrep.*\b(flood|attack).*then.*start', "DDoS process check/restart"), ] for pattern, description in respawn_patterns: if cls._safe_regex_search(pattern, content, re.IGNORECASE | re.DOTALL): threats.append({ "type": "ddos_persistence_respawn", "description": f"DDoS respawn/watchdog: {description} in {filename}", "severity": "critical", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass if checked > 300: break except Exception as e: logger.debug(f"DDoS respawn scan error: {e}") # 4. Check systemd user services rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /etc/systemd/system/*.service /home/*/.config/systemd/user/*.service " f"/root/.config/systemd/user/*.service 2>/dev/null'", timeout=5, ) if rc == 0 and out: if re.search(r'(flood|attack|ddos|stress|hping|xerxes|loic|mirai|gafgyt|booter|stresser)', out, re.IGNORECASE): threats.append({ "type": "ddos_persistence_systemd", "description": "DDoS tool configured as systemd service", "severity": "critical", "evidence": out.strip()[:300], }) return threats @classmethod def _scan_encoded_ddos_payloads(cls, uuid): """Scan for base64/hex encoded DDoS attack payloads. DDoS tools are often distributed as base64-encoded scripts to evade simple file-based detection. This method: 1. Finds base64 blobs in script/config files 2. Decodes them and checks for DDoS patterns 3. Also looks for hex-encoded attack payloads 4. Scans for environment variables with encoded attack configs """ threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): return threats import base64 as b64_mod ddos_decoded_patterns = [ r'socket\.SOCK_RAW|IPPROTO_RAW|AF_PACKET', r'sendto.*flood|flood.*sendto', r'(syn|udp|tcp|icmp|http)[_-]?flood', r'amplif(y|ication)|reflect(or|ion)', r'target[_-]?(ip|host|port)|victim[_-]?(ip|host)', r'attack[_-]?(method|type|target)|ddos', r'botnet|zombie|c2.server|cnc', r'hping|nping|scapy|masscan', r'SOCK_DGRAM.*sendto|socket.*send.*while', r'import\s+socket.*for.*range.*send', r'(mirai|gafgyt|bashlite|kaiten)', r'--flood|--rand-source|--syn', ] try: checked = 0 scan_exts = {".py", ".sh", ".bash", ".pl", ".rb", ".js", ".php", ".json", ".yaml", ".yml", ".conf", ".cfg", ".txt", ".env"} for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps"): dirs.clear() continue for filename in files: if checked > 400: break checked += 1 entry = Path(root) / filename suffix = entry.suffix.lower() if suffix not in scan_exts: continue try: if entry.stat().st_size > 512000: continue content = entry.read_text(errors="ignore")[:32768] # Find base64 blobs (40+ chars) b64_blobs = re.findall(r'[A-Za-z0-9+/]{40,}={0,2}', content) for blob in b64_blobs[:10]: try: decoded = b64_mod.b64decode(blob).decode("utf-8", errors="ignore") ddos_matches = [] for pattern in ddos_decoded_patterns: if re.search(pattern, decoded, re.IGNORECASE): ddos_matches.append(pattern.split("|")[0][:25]) if len(ddos_matches) >= 2: threats.append({ "type": "ddos_encoded_payload", "description": (f"Base64-encoded DDoS payload in {filename} " f"({len(ddos_matches)} attack patterns)"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Decoded patterns: {ddos_matches[:5]}", }) except Exception: pass # Find hex-encoded blobs (long hex strings) hex_blobs = re.findall(r'(?:0x)?([0-9a-fA-F]{80,})', content) for blob in hex_blobs[:5]: try: decoded = bytes.fromhex(blob).decode("utf-8", errors="ignore") for pattern in ddos_decoded_patterns: if cls._safe_regex_search(pattern, decoded, re.IGNORECASE): threats.append({ "type": "ddos_hex_encoded_payload", "description": f"Hex-encoded DDoS payload in {filename}", "severity": "critical", "path": str(entry.relative_to(data_dir)), }) break except Exception: pass # Check for env-style DDoS configuration variables env_ddos_patterns = [ r'(TARGET|VICTIM|ATTACK|FLOOD|DDOS|STRESS)[_-]?(IP|HOST|PORT|URL|METHOD|DURATION)\s*=', r'(C2|CNC|BOT|IRC)[_-]?(SERVER|HOST|CHANNEL|PORT)\s*=', r'(STRESSER|BOOTER)[_-]?(API|KEY|URL)\s*=', r'(THREAD|WORKER|PROCESS)[_-]?COUNT\s*=\s*\d{2,}', r'(PACKET|CONNECTION)[_-]?COUNT\s*=\s*\d{4,}', ] env_hits = [] for pattern in env_ddos_patterns: if re.search(pattern, content, re.IGNORECASE): env_hits.append(pattern.split("[")[0][:20]) if len(env_hits) >= 2: threats.append({ "type": "ddos_config_file", "description": (f"DDoS configuration file: {filename} " f"({len(env_hits)} attack config variables)"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Config vars: {env_hits[:5]}", }) except (OSError, PermissionError): pass if checked > 400: break except Exception as e: logger.debug(f"Encoded DDoS payload scan error: {e}") return threats # -------------------------------------------------------------- # Phase 3: Deep Miner Detection (Anti-Obfuscation / Anti-Wrapper) # -------------------------------------------------------------- @classmethod def _scan_binary_strings(cls, uuid): """Read ELF binary content and scan for miner signature strings. This catches miners even when the binary is renamed to "bash", "java", "update-service", etc. The compiled binary still contains miner-specific strings like "stratum+tcp://", "RandomX", "hashrate", etc. Also scans for wallet addresses embedded in binaries. """ threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): return threats try: checked_binaries = 0 max_binaries = 50 for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps", "jre", "jdk", "lib"): dirs.clear() continue for filename in files: if checked_binaries >= max_binaries: break entry = Path(root) / filename try: # Only check executable files or files without extension if not os.access(str(entry), os.X_OK) and entry.suffix: continue # Skip large files (>50MB) and tiny files (<10KB) fsize = entry.stat().st_size if fsize > 50 * 1024 * 1024 or fsize < 10240: continue # Check ELF magic with open(entry, "rb") as f: magic = f.read(4) if magic != b'\x7fELF': continue checked_binaries += 1 # Read binary in chunks, scan for miner strings f.seek(0) miner_string_hits = [] wallet_hits = [] chunk_size = 256 * 1024 # 256KB chunks bytes_read = 0 max_read = 10 * 1024 * 1024 # Max 10MB per binary while bytes_read < max_read: chunk = f.read(chunk_size) if not chunk: break bytes_read += len(chunk) # Check for miner signature strings for sig in cls.MINER_BINARY_STRINGS: if sig in chunk and sig.decode("ascii", errors="ignore") not in [h[0] for h in miner_string_hits]: miner_string_hits.append((sig.decode("ascii", errors="ignore"), len(miner_string_hits))) # Check for wallet addresses in printable string regions # Extract printable strings (4+ chars) printable = re.findall(rb'[\x20-\x7e]{20,}', chunk) for pstr in printable: try: text = pstr.decode("ascii", errors="ignore") for pattern, wallet_type in cls.WALLET_PATTERNS: if re.search(pattern, text) and wallet_type not in [w[0] for w in wallet_hits]: wallet_hits.append((wallet_type, text[:80])) except Exception: pass # Evaluate results if len(miner_string_hits) >= 3: hit_names = [h[0][:40] for h in miner_string_hits[:8]] threats.append({ "type": "miner_binary_strings", "description": (f"Crypto miner binary detected (renamed): {filename} " f"({len(miner_string_hits)} miner signatures inside ELF)"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Strings found: {hit_names}", }) elif len(miner_string_hits) >= 1: threats.append({ "type": "suspicious_binary_strings", "description": (f"Suspicious binary: {filename} " f"({len(miner_string_hits)} mining-related string(s))"), "severity": "suspicious", "path": str(entry.relative_to(data_dir)), "evidence": f"Strings: {[h[0][:40] for h in miner_string_hits[:5]]}", }) if wallet_hits: threats.append({ "type": "wallet_in_binary", "description": (f"Cryptocurrency wallet address found in binary: {filename} " f"({', '.join(w[0] for w in wallet_hits[:3])})"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Wallets: {[w[1][:50] for w in wallet_hits[:3]]}", }) except (OSError, PermissionError): pass if checked_binaries >= max_binaries: break except Exception as e: logger.warning(f"[{uuid}] Binary string scan error: {e}", exc_info=True) if threats: logger.warning(f"[{uuid}] Binary scan found {len(threats)} threat(s)") return threats @classmethod def _detect_wrapper_miners(cls, uuid, container_name): """Detect miners hidden inside shell/python/node wrapper scripts. Common evasion: user writes a bash script that: - Downloads miner at runtime (curl/wget) - Decodes miner from base64 blob inside the script - Launches miner from /tmp, /dev/shm, or hidden directory - Renames miner process via exec -a "java" ./miner - Uses nohup/setsid/disown to detach from terminal """ threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): return threats wrapper_indicators = [ # Download + execute patterns (r'(curl|wget)\s+[^\n]*\|\s*(ba)?sh', "Download and pipe to shell"), (r'(curl|wget)\s+[^\n]*(chmod|\.\/)', "Download and execute"), (r'(curl|wget)\s+[^\n]*(/tmp/|/dev/shm/|/var/tmp/)', "Download to temp directory"), (r'(curl|wget)\s+[^\n]*(xmrig|miner|pool|stratum)', "Download miner directly"), # Base64 decode + execute (r'base64\s+(-d|--decode)\s*[^\n]*(chmod|\.\/)' , "Base64 decode and execute"), (r'echo\s+[^\n]*\|\s*base64\s+(-d|--decode)\s*>\s*[^\n]*(\.sh|\.bin|/tmp/)', "Base64 decode to file"), (r'python[23]?\s+-c\s+[^\n]*base64[^\n]*(exec|eval|subprocess)', "Python base64 exec"), (r'node\s+-e\s+[^\n]*(Buffer\.from|atob)[^\n]*(exec|spawn|child_process)', "Node.js base64 exec"), # Process renaming / hiding (r'exec\s+-a\s+["\'][^"\']+["\']\s+\./', "Process rename via exec -a"), (r'prctl\s*\(\s*PR_SET_NAME', "Process rename via prctl"), (r'LD_PRELOAD\s*=', "LD_PRELOAD library injection"), # Detach from terminal (r'nohup\s+[^\n]*(miner|xmr|pool|stratum|\.\/[a-z]{1,5})\b', "nohup miner launch"), (r'setsid\s+[^\n]*(miner|xmr|\.\/)', "setsid miner detach"), (r'disown\b', "Process disowned from shell"), # Hidden directory execution (r'\./\.[a-z]+/', "Execution from hidden directory"), (r'/\.\w+/[^\s]+\s', "Binary in hidden directory"), # Obfuscated variable names hiding miner commands (r'\$\{?\w*(pool|mine|hash|coin|algo|stratum|wallet)\w*\}?', "Variable name referencing mining"), # Hex/octal escape execution (r'\\x[0-9a-f]{2}.*\\x[0-9a-f]{2}.*exec', "Hex-escaped command execution"), (r'\$\(printf\s+[^\)]*\\\\x', "Printf hex escape execution"), (r'echo\s+-e\s+[^\n]*\\\\x[0-9a-f]', "Echo hex escape"), ] try: script_exts = {".sh", ".bash", ".py", ".js", ".php", ".pl", ".rb", ".ts"} checked = 0 max_files = 500 for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 5: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps"): dirs.clear() continue for filename in files: if checked >= max_files: break checked += 1 entry = Path(root) / filename suffix = entry.suffix.lower() # Check scripts and also extensionless executable files is_script = suffix in script_exts is_exec_no_ext = not entry.suffix and os.access(str(entry), os.X_OK) if not is_script and not is_exec_no_ext: continue try: fsize = entry.stat().st_size if fsize > 1024 * 1024 or fsize < 10: # Skip >1MB and <10B continue content = entry.read_text(errors="ignore")[:32768] wrapper_hits = [] for pattern, description in wrapper_indicators: if cls._safe_regex_search(pattern, content, re.IGNORECASE): wrapper_hits.append(description) # Also check for mining pool URLs in the script for pool in cls.MINING_POOL_DOMAINS: if pool in content.lower(): wrapper_hits.append(f"Pool domain: {pool}") # Check for wallet addresses in scripts for pattern, wallet_type in cls.WALLET_PATTERNS: if cls._safe_regex_search(pattern, content): wrapper_hits.append(f"Wallet: {wallet_type}") if len(wrapper_hits) >= 2: threats.append({ "type": "miner_wrapper_script", "description": (f"Miner wrapper script detected: {filename} " f"({len(wrapper_hits)} indicators)"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Indicators: {wrapper_hits[:6]}", }) elif len(wrapper_hits) == 1: threats.append({ "type": "suspicious_wrapper_script", "description": f"Suspicious script: {filename} ({wrapper_hits[0]})", "severity": "suspicious", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass if checked >= max_files: break except Exception as e: logger.warning(f"[{uuid}] Wrapper scan error: {e}", exc_info=True) # Also check running shell processes for inline miner commands rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/cmdline; do echo \"==$p==\"; " f"cat \"$p\" 2>/dev/null | tr \"\\0\" \" \"; echo; done'", timeout=15, ) if rc == 0 and out: for line in out.splitlines(): if line.startswith("==") or not line.strip(): continue line_lower = line.lower() # Detect: bash -c "curl ... | sh", python -c "import ... exec" if re.search(r'(ba)?sh\s+-c\s+.*\b(curl|wget)\b.*\b(sh|bash|python)\b', line, re.IGNORECASE): threats.append({ "type": "runtime_download_execute", "description": "Live download-and-execute detected in running process", "severity": "critical", "evidence": line.strip()[:200], }) # Detect: exec -a "java" ./something if re.search(r'exec\s+-a\s+["\']', line, re.IGNORECASE): threats.append({ "type": "process_rename_live", "description": "Live process rename (exec -a) - hiding real binary name", "severity": "critical", "evidence": line.strip()[:200], }) # Detect base64 decode piped to execution if re.search(r'base64.*-d.*\|\s*(ba)?sh', line, re.IGNORECASE): threats.append({ "type": "base64_execute_live", "description": "Live base64-decode-and-execute in process", "severity": "critical", "evidence": line.strip()[:200], }) return threats @classmethod def _detect_fileless_miners(cls, container_name): """Detect fileless miner execution methods. Miners can run without any file on disk using: - memfd_create() - creates anonymous file in memory, gets fd - /dev/shm/ - shared memory filesystem (tmpfs) - /proc/self/fd/ execution - running from file descriptors - /tmp/ with deleted files - download, run, delete These leave no trace on the filesystem but are visible in /proc. """ threats = [] # Check /proc/[pid]/exe for memfd or deleted binaries rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/exe; do " f"target=$(readlink \"$p\" 2>/dev/null); " f"if [ -n \"$target\" ]; then echo \"$p -> $target\"; fi; " f"done'", timeout=15, ) if rc == 0 and out: for line in out.splitlines(): if not line.strip(): continue line_lower = line.lower() # memfd_create - anonymous memory-only execution if "memfd:" in line_lower or "/memfd:" in line_lower: threats.append({ "type": "fileless_memfd_execution", "description": "Fileless execution via memfd_create() - binary running from memory only", "severity": "critical", "evidence": line.strip()[:200], }) # Running from /dev/shm (shared memory) if "/dev/shm/" in line: threats.append({ "type": "fileless_shm_execution", "description": "Binary running from /dev/shm/ (shared memory) - common miner hiding technique", "severity": "critical", "evidence": line.strip()[:200], }) # Running from /tmp with deleted binary if "/tmp/" in line and "(deleted)" in line: threats.append({ "type": "fileless_tmp_deleted", "description": "Deleted binary running from /tmp/ - download-run-delete pattern", "severity": "critical", "evidence": line.strip()[:200], }) # Running from /var/tmp if "/var/tmp/" in line: threats.append({ "type": "suspicious_vartmp_execution", "description": "Binary running from /var/tmp/ - suspicious execution location", "severity": "suspicious", "evidence": line.strip()[:200], }) # Running from /proc/self/fd if "/proc/self/fd/" in line or "/proc/" in line and "/fd/" in line: if "memfd:" not in line_lower: # memfd already caught above threats.append({ "type": "fileless_fd_execution", "description": "Binary running from /proc/fd - file descriptor execution", "severity": "critical", "evidence": line.strip()[:200], }) # Check for files in /dev/shm rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ls -la /dev/shm/ 2>/dev/null'", timeout=5, ) if rc == 0 and out: for line in out.splitlines(): if line.startswith("total") or not line.strip(): continue # Any executable file in /dev/shm is suspicious if line.startswith("-") and ("x" in line[:10]): threats.append({ "type": "executable_in_shm", "description": f"Executable file in /dev/shm/ - likely hidden miner or malware", "severity": "critical", "evidence": line.strip()[:200], }) # Check /proc/[pid]/maps for suspicious memory-mapped regions rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/maps; do " f"grep -l \"memfd:\\|/dev/shm/\\|/dev/zero\" \"$p\" 2>/dev/null; done'", timeout=10, ) if rc == 0 and out: pids_with_memfd = set() for line in out.splitlines(): if "/proc/" in line and "/maps" in line: pid = line.split("/")[2] if len(line.split("/")) > 2 else "" if pid.isdigit(): pids_with_memfd.add(pid) if pids_with_memfd: threats.append({ "type": "suspicious_memory_maps", "description": (f"Processes with suspicious memory mappings (memfd/shm): " f"PIDs {list(pids_with_memfd)[:10]}"), "severity": "suspicious", "evidence": f"PIDs: {list(pids_with_memfd)[:10]}", }) return threats @classmethod def _detect_miner_persistence(cls, uuid, container_name): """Detect persistence mechanisms that re-spawn miners after kill. Scans for: - Crontab entries with suspicious commands - at jobs - systemd user services - .bashrc/.profile/.bash_profile injection - init.d scripts - Supervisor configs - Loop scripts (while true; do ... done) """ threats = [] # Check crontab rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'crontab -l 2>/dev/null; cat /etc/crontab 2>/dev/null; " f"cat /var/spool/cron/crontabs/* 2>/dev/null; " f"cat /etc/cron.d/* 2>/dev/null'", timeout=10, ) if rc == 0 and out: for line in out.splitlines(): if line.startswith("#") or not line.strip(): continue line_lower = line.lower() suspicious_cron = False reason = "" # Check for miner-related content if any(kw in line_lower for kw in ["miner", "xmrig", "stratum", "pool.", "mining", "hashrate", "/tmp/.", "/dev/shm/", "curl.*sh", "wget.*sh", "base64.*-d"]): suspicious_cron = True reason = "miner-related command in crontab" # Check for download-execute pattern if re.search(r'(curl|wget)\s+[^\n]*\|\s*(ba)?sh', line, re.IGNORECASE): suspicious_cron = True reason = "download-and-execute in crontab" # Check for hidden file execution if re.search(r'/\.+[a-z]+', line) and ("/" in line): suspicious_cron = True reason = "hidden file/directory in crontab" if suspicious_cron: threats.append({ "type": "miner_persistence_cron", "description": f"Miner persistence via cron: {reason}", "severity": "critical", "evidence": line.strip()[:200], }) # Check .bashrc, .profile, .bash_profile for miner injection rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /home/*/.bashrc /home/*/.profile /home/*/.bash_profile " f"/root/.bashrc /root/.profile /root/.bash_profile " f"/container/.bashrc 2>/dev/null'", timeout=10, ) if rc == 0 and out: for line in out.splitlines(): line_lower = line.lower() if line.startswith("#") or not line.strip(): continue # Look for miner auto-start in shell profiles if any(kw in line_lower for kw in ["miner", "xmrig", "stratum", "pool.", "mining", "/dev/shm/", "nohup.*./", "base64.*-d.*sh"]): threats.append({ "type": "miner_persistence_profile", "description": "Miner auto-start injected into shell profile (.bashrc/.profile)", "severity": "critical", "evidence": line.strip()[:200], }) # Generic download-execute in profile if re.search(r'(curl|wget)\s+[^\n]*\|\s*(ba)?sh', line, re.IGNORECASE): threats.append({ "type": "miner_persistence_profile", "description": "Download-and-execute in shell profile (possible miner dropper)", "severity": "critical", "evidence": line.strip()[:200], }) # Check for respawn loop scripts in data directory data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if data_dir.exists(): try: for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 3: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam"): dirs.clear() continue for filename in files: entry = Path(root) / filename if entry.suffix.lower() not in (".sh", ".bash", ".py", ""): continue try: if entry.stat().st_size > 50000: continue content = entry.read_text(errors="ignore")[:8192] # Detect respawn loops: "while true; do ./miner; done" if re.search(r'while\s+(true|1|\:)\s*[;\n]\s*do\b', content, re.IGNORECASE): loop_content = content.lower() if any(kw in loop_content for kw in ["miner", "xmrig", "pool", "stratum", "hash", "./", "/tmp/", "/dev/shm/"]): threats.append({ "type": "miner_respawn_loop", "description": f"Miner respawn loop script: {filename} (while true + miner execution)", "severity": "critical", "path": str(entry.relative_to(data_dir)), }) except (OSError, PermissionError): pass except Exception: pass # Check systemd user services rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'find /etc/systemd /home/*/.config/systemd /root/.config/systemd " f"-name \"*.service\" -type f 2>/dev/null -exec cat {{}} \\;'", timeout=10, ) if rc == 0 and out: out_lower = out.lower() if any(kw in out_lower for kw in ["miner", "xmrig", "stratum", "pool.", "mining", "/dev/shm/", "/tmp/."]): threats.append({ "type": "miner_persistence_systemd", "description": "Miner persistence via systemd service", "severity": "critical", "evidence": out.strip()[:300], }) return threats @classmethod def _correlate_cpu_with_processes(cls, container_name): """Cross-reference high-CPU processes with their binary identity. If a process named "bash" or "java" is consuming >50% CPU, read its /proc/pid/exe to see what the REAL binary is. Renamed miners show innocent names but /proc/pid/exe reveals the truth. """ threats = [] # Get top CPU-consuming processes rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ps -eo pid,pcpu,comm --sort=-pcpu 2>/dev/null | head -20'", timeout=10, ) if rc != 0 or not out: return threats high_cpu_pids = [] for line in out.splitlines()[1:]: # Skip header parts = line.split() if len(parts) < 3: continue try: pid = parts[0] cpu = float(parts[1]) comm = parts[2] if cpu >= 30.0: # Any process using >30% CPU high_cpu_pids.append((pid, cpu, comm)) except (ValueError, IndexError): pass for pid, cpu, comm in high_cpu_pids: # Read the real binary path rc2, exe_path, _ = run_cmd( f"docker exec {container_name} readlink -f /proc/{pid}/exe 2>/dev/null", timeout=5, ) if rc2 != 0 or not exe_path: continue exe_path = exe_path.strip() # Read cmdline for more context rc3, cmdline, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /proc/{pid}/cmdline 2>/dev/null | tr \"\\0\" \" \"'", timeout=5, ) cmdline = (cmdline or "").strip() # Check if process name doesn't match the binary exe_name = os.path.basename(exe_path).lower() comm_lower = comm.lower() # Known safe high-CPU processes for game servers safe_names = {"java", "node", "python", "python3", "srcds_linux", "hlds_linux", "bedrock_server", "palserver", "factoryserver", "7daystodieserver", "dayzserver", "rustserver", "valheim", "unturned", "terraria", "tmodloader"} if comm_lower in safe_names and exe_name in safe_names: continue # Legitimate high-CPU game server # Flag 1: process says "bash" but binary is not /bin/bash if comm_lower in ("bash", "sh", "dash", "zsh") and not exe_path.startswith(("/bin/", "/usr/bin/")): threats.append({ "type": "renamed_high_cpu_process", "description": (f"Process claims to be '{comm}' but binary is {exe_path} " f"(CPU: {cpu}%) - likely renamed miner"), "severity": "critical", "evidence": f"PID {pid}: comm={comm}, exe={exe_path}, cpu={cpu}%, cmdline={cmdline[:150]}", }) # Flag 2: high CPU process with unknown/suspicious binary elif exe_name not in safe_names and cpu >= 50.0: # Check miner patterns in the binary/cmdline combined = f"{exe_path} {cmdline}".lower() miner_kw = ["miner", "xmrig", "pool", "stratum", "hash", "crypto", "randomx", "cryptonight", "kawpow", "ethash"] if any(kw in combined for kw in miner_kw): threats.append({ "type": "high_cpu_miner_process", "description": (f"High-CPU process with miner keywords: PID {pid} " f"({exe_name}, CPU: {cpu}%)"), "severity": "critical", "evidence": f"exe={exe_path}, cpu={cpu}%, cmdline={cmdline[:150]}", }) elif "(deleted)" in exe_path: threats.append({ "type": "high_cpu_deleted_binary", "description": (f"High-CPU process running deleted binary: PID {pid} " f"(CPU: {cpu}%) - classic miner concealment"), "severity": "critical", "evidence": f"exe={exe_path}, cpu={cpu}%", }) elif not any(ok in exe_path for ok in ["/usr/", "/bin/", "/opt/", "/lib/"]): threats.append({ "type": "high_cpu_unknown_binary", "description": (f"High CPU ({cpu}%) from unknown binary: {exe_path} (PID {pid}, " f"name: {comm}) - investigate"), "severity": "suspicious", "evidence": f"exe={exe_path}, cpu={cpu}%, cmdline={cmdline[:150]}", }) return threats @classmethod def _scan_encoded_configs(cls, uuid): """Scan for base64/hex-encoded mining pool URLs and wallet addresses. Miners often hide their configs by encoding pool URLs and wallet addresses in base64 or hex to evade pattern matching. We decode and re-scan the content. """ threats = [] data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): return threats import base64 try: checked = 0 max_files = 300 scan_exts = {".sh", ".bash", ".py", ".js", ".php", ".json", ".txt", ".conf", ".cfg", ".yaml", ".yml", ".toml", ".env", ".ini", ".xml", ".pl", ".rb", ".ts", ""} for root, dirs, files in os.walk(str(data_dir)): depth = str(root).replace(str(data_dir), "").count(os.sep) if depth > 4: dirs.clear() continue dir_name = os.path.basename(root) if dir_name in ("node_modules", ".git", "__pycache__", "steamcmd", "Steam", "steamapps", "lib", "jre"): dirs.clear() continue for filename in files: if checked >= max_files: break entry = Path(root) / filename suffix = entry.suffix.lower() if suffix not in scan_exts: continue try: fsize = entry.stat().st_size if fsize > 512 * 1024 or fsize < 20: # Skip >512KB and <20B continue checked += 1 content = entry.read_text(errors="ignore")[:65536] # Find base64 blobs (40+ chars, no spaces) b64_pattern = re.findall(r'[A-Za-z0-9+/]{40,}={0,3}', content) for blob in b64_pattern[:20]: # Max 20 blobs per file try: decoded = base64.b64decode(blob).decode("utf-8", errors="ignore") decoded_lower = decoded.lower() # Check decoded content for mining indicators hits = [] if "stratum" in decoded_lower: hits.append("stratum URL") if any(pool in decoded_lower for pool in cls.MINING_POOL_DOMAINS[:20]): hits.append("pool domain") for wp, wt in cls.WALLET_PATTERNS: if re.search(wp, decoded): hits.append(wt) break if any(kw in decoded_lower for kw in ["xmrig", "miner", "mining", "hashrate", "donate-level"]): hits.append("miner keyword") for p in cls.MINER_CONFIG_PATTERNS[:10]: if re.search(p, decoded, re.IGNORECASE): hits.append("miner config pattern") break if hits: threats.append({ "type": "encoded_miner_config", "description": (f"Base64-encoded miner data in {filename}: " f"{', '.join(hits[:4])}"), "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Decoded: {decoded[:120]}", }) except Exception: pass # Find hex-encoded strings (long hex sequences) hex_pattern = re.findall(r'(?:0x)?([0-9a-fA-F]{60,})', content) for blob in hex_pattern[:10]: try: decoded = bytes.fromhex(blob).decode("utf-8", errors="ignore") decoded_lower = decoded.lower() if any(kw in decoded_lower for kw in ["stratum", "pool.", "miner", "xmrig", "mining", "hashrate"]): threats.append({ "type": "hex_encoded_miner_config", "description": f"Hex-encoded miner data in {filename}", "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Decoded: {decoded[:120]}", }) except Exception: pass # Check for environment-style variables with encoded values env_pattern = re.findall(r'(?:POOL|WALLET|MINING|ALGO|COIN|URL|WORKER|HOST)\s*=\s*["\']?([^\s"\']+)', content, re.IGNORECASE) for val in env_pattern: val_lower = val.lower() if any(kw in val_lower for kw in ["stratum", "pool.", "nicehash", "monero", "mining", "hash"]): threats.append({ "type": "miner_env_variable", "description": f"Mining-related environment variable in {filename}", "severity": "critical", "path": str(entry.relative_to(data_dir)), "evidence": f"Value: {val[:100]}", }) except (OSError, PermissionError): pass if checked >= max_files: break except Exception as e: logger.warning(f"[{uuid}] Encoded config scan error: {e}", exc_info=True) return threats @classmethod def _deep_network_inspection(cls, container_name): """Deep network analysis for encrypted/proxied mining connections. Detects: - DNS queries to mining pool domains (even if connection is encrypted) - Environment variables with pool URLs (set by wrapper scripts) - SOCKS/HTTP proxy connections that might tunnel to pools - TLS connections to known mining ports - /etc/hosts manipulation for pool domain resolution """ threats = [] # 1. Check /etc/resolv.conf and /etc/hosts for pool-related manipulation rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /etc/hosts 2>/dev/null'", timeout=5, ) if rc == 0 and out: for line in out.splitlines(): line_lower = line.lower() if line.startswith("#") or not line.strip(): continue for pool in cls.MINING_POOL_DOMAINS[:30]: if pool in line_lower: threats.append({ "type": "hosts_file_pool_entry", "description": f"Mining pool domain in /etc/hosts: {pool}", "severity": "critical", "evidence": line.strip()[:200], }) # 2. Scan ALL environment variables of ALL processes for pool URLs rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'for p in /proc/[0-9]*/environ; do " f"cat \"$p\" 2>/dev/null | tr \"\\0\" \"\\n\"; done'", timeout=15, ) if rc == 0 and out: seen_env_threats = set() for line in out.splitlines(): if not line.strip() or "=" not in line: continue line_lower = line.lower() # Check for pool URLs in any env var if "stratum" in line_lower or "stratum+tcp" in line_lower or "stratum+ssl" in line_lower: key = "stratum_env" if key not in seen_env_threats: seen_env_threats.add(key) threats.append({ "type": "miner_env_pool_url", "description": "Mining pool stratum URL found in process environment", "severity": "critical", "evidence": line.strip()[:200], }) # Check for pool domains in env vars for pool in cls.MINING_POOL_DOMAINS[:30]: if pool in line_lower: key = f"pool_{pool}" if key not in seen_env_threats: seen_env_threats.add(key) threats.append({ "type": "miner_env_pool_domain", "description": f"Mining pool domain in environment variable: {pool}", "severity": "critical", "evidence": line.strip()[:200], }) # Check for wallet addresses in env vars for wp, wt in cls.WALLET_PATTERNS: if re.search(wp, line): key = f"wallet_{wt}" if key not in seen_env_threats: seen_env_threats.add(key) threats.append({ "type": "miner_env_wallet", "description": f"Cryptocurrency wallet in environment: {wt}", "severity": "critical", "evidence": line.strip()[:200], }) break # Check for mining-related env var names env_key = line.split("=", 1)[0].upper() mining_env_names = ["POOL", "MINING_POOL", "POOL_URL", "POOL_PASS", "WALLET", "WALLET_ADDRESS", "MINER_URL", "ALGO", "COIN", "WORKER", "RIG_ID", "DONATE_LEVEL", "XMRIG_", "MINING_", "HASHRATE", "STRATUM"] for menv in mining_env_names: if menv in env_key: key = f"envname_{menv}" if key not in seen_env_threats: seen_env_threats.add(key) threats.append({ "type": "miner_env_variable_name", "description": f"Mining-related environment variable: {env_key}", "severity": "suspicious", "evidence": line.strip()[:200], }) # 3. Check for proxy/tunnel connections that might hide mining traffic rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ps aux 2>/dev/null'", timeout=10, ) if rc == 0 and out: proxy_patterns = [ (r'\bssh\s+.*-[LRD]\s', "SSH tunnel (possible mining traffic tunnel)"), (r'\bssh\s+.*-f\s.*-N\s', "SSH background tunnel"), (r'\bsocat\b.*TCP.*TCP', "Socat TCP relay (possible pool proxy)"), (r'\bsocat\b.*SOCKS', "Socat SOCKS proxy"), (r'\bchisel\b', "Chisel tunnel (reverse proxy for mining)"), (r'\bfrpc?\b', "FRP tunnel (fast reverse proxy)"), (r'\bngrok\b', "Ngrok tunnel"), (r'\bcloudflared\b', "Cloudflare tunnel"), (r'\bsocks[45]?\b.*proxy', "SOCKS proxy server"), (r'\bproxychains\b', "ProxyChains (hiding connections through proxies)"), (r'\btorsocks\b', "Tor SOCKS (anonymizing miner connections)"), ] for pattern, description in proxy_patterns: if re.search(pattern, out, re.IGNORECASE): threats.append({ "type": "suspicious_tunnel_proxy", "description": description, "severity": "suspicious", "evidence": re.search(pattern, out, re.IGNORECASE).group()[:200], }) # 4. Check DNS cache / recent DNS queries for pool domains rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'cat /etc/resolv.conf 2>/dev/null; " f"getent hosts pool.minexmr.com 2>/dev/null; " f"getent hosts pool.supportxmr.com 2>/dev/null; " f"getent hosts gulf.moneroocean.stream 2>/dev/null; " f"getent hosts pool.hashvault.pro 2>/dev/null; " f"getent hosts auto.c3pool.org 2>/dev/null; " f"getent hosts xmr.2miners.com 2>/dev/null; " f"getent hosts pool.kryptex.com 2>/dev/null; " f"getent hosts rx.unmineable.com 2>/dev/null; " f"getent hosts randomxmonero.xmrig.com 2>/dev/null'", timeout=10, ) # If getent succeeds for any pool domain, it means DNS was resolved (likely in use) if rc == 0 and out: pool_domains = ["minexmr", "supportxmr", "moneroocean", "hashvault", "c3pool", "2miners", "kryptex", "unmineable", "xmrig"] for pool in pool_domains: if pool in out.lower() and re.search(r'\d+\.\d+\.\d+\.\d+', out): threats.append({ "type": "dns_pool_resolution", "description": f"Mining pool DNS resolved: {pool} - active pool connection likely", "severity": "critical", "evidence": f"DNS resolved for: {pool}", }) # 5. Check established TLS connections on mining ports rc, out, _ = run_cmd( f"docker exec {container_name} sh -c " f"'ss -tnp 2>/dev/null | grep -E \"(ESTAB|ESTABLISHED)\" | grep -E \":(443|3333|4444|5555|14443|14444|10128|45700|9999)\"'", timeout=5, ) if rc == 0 and out and out.strip(): tls_mining_ports = {"443": "TLS", "3333": "common", "4444": "common", "5555": "common", "14443": "stratum-tls", "14444": "stratum-tls", "10128": "XMRig", "45700": "MoneroOcean", "9999": "common"} for line in out.splitlines(): for port, ptype in tls_mining_ports.items(): if f":{port}" in line and port != "443": # Port 443 alone is not suspicious threats.append({ "type": "tls_mining_connection", "description": f"TLS connection to mining port :{port} ({ptype})", "severity": "suspicious", "evidence": line.strip()[:200], }) return threats # --- Server Type Detector ---------------------------------------- class ServerTypeDetector: """Automatically detect game server type from egg configuration. Supports: SteamCMD, LinuxGSM, Custom (generic). The agent uses the detected type to choose the correct install/start pipeline. """ STEAMCMD = "steamcmd" LINUXGSM = "linuxgsm" CUSTOM = "custom" @classmethod def detect(cls, startup_command, environment, install_script="", docker_image=""): """Detect server type. Returns 'steamcmd', 'linuxgsm', or 'custom'.""" if _is_linuxgsm_server(startup_command, environment): return cls.LINUXGSM if cls._is_steamcmd_server(environment, install_script): return cls.STEAMCMD return cls.CUSTOM @classmethod def _is_steamcmd_server(cls, environment, install_script=""): """Detect if this server uses SteamCMD.""" steam_app_id = cls.get_steam_app_id(environment, install_script) if steam_app_id: return True if install_script: lower = install_script.lower() if "steamcmd" in lower or "+app_update" in lower: return True return False @classmethod def get_steam_app_id(cls, environment, install_script=""): """Extract Steam App ID from environment or install script.""" for key in ("STEAM_APP_ID", "SRCDS_APPID", "STEAM_APPID"): val = environment.get(key, "") if val and str(val).strip().isdigit(): return str(val).strip() if install_script: match = re.search(r'app_update\s+(\d+)', install_script) if match: return match.group(1) return None logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ logging.StreamHandler(sys.stdout), ], ) logger = logging.getLogger("game-agent") def run_cmd(cmd, timeout=30): """Run a shell command and return (returncode, stdout, stderr).""" try: result = subprocess.run( cmd, shell=True, capture_output=True, text=True, timeout=timeout ) return result.returncode, result.stdout.strip(), result.stderr.strip() except subprocess.TimeoutExpired: return -1, "", "Command timed out" except Exception as e: return -1, "", str(e) def get_container_name(uuid): """Get the Docker container name for a game server UUID.""" safe = re.sub(r'[^a-zA-Z0-9_-]', '', uuid) return f"gs-{safe}" def get_system_metrics(): """Collect system metrics like CPU, memory, disk usage.""" metrics = { "agent_version": AGENT_VERSION, "os_name": "", "kernel_version": "", "uptime_seconds": None, "cpu_load": None, "memory_total_mb": None, "memory_used_mb": None, "disk_total_mb": None, "disk_used_mb": None, "docker_running": False, "servers_running": 0, } # OS info rc, out, _ = run_cmd("uname -o 2>/dev/null") metrics["os_name"] = out if rc == 0 else "Linux" rc, out, _ = run_cmd("uname -r 2>/dev/null") if rc == 0: metrics["kernel_version"] = out # Uptime try: with open("/proc/uptime") as f: metrics["uptime_seconds"] = int(float(f.read().split()[0])) except Exception: pass # CPU load try: metrics["cpu_load"] = round(os.getloadavg()[0], 2) except Exception: pass # Memory try: with open("/proc/meminfo") as f: meminfo = {} for line in f: parts = line.split() if len(parts) >= 2: key = parts[0].rstrip(":") meminfo[key] = int(parts[1]) if "MemTotal" in meminfo: metrics["memory_total_mb"] = meminfo["MemTotal"] // 1024 if "MemTotal" in meminfo and "MemAvailable" in meminfo: metrics["memory_used_mb"] = (meminfo["MemTotal"] - meminfo["MemAvailable"]) // 1024 except Exception: pass # Disk try: st = os.statvfs("/") total = st.f_blocks * st.f_frsize free = st.f_bfree * st.f_frsize metrics["disk_total_mb"] = total // (1024 * 1024) metrics["disk_used_mb"] = (total - free) // (1024 * 1024) except Exception: pass # Docker rc, _, _ = run_cmd("docker info >/dev/null 2>&1") metrics["docker_running"] = rc == 0 # Count running game containers if metrics["docker_running"]: rc, out, _ = run_cmd('docker ps --filter "name=gs-" --format "{{.Names}}" 2>/dev/null') if rc == 0 and out: metrics["servers_running"] = len(out.strip().splitlines()) return metrics def docker_server_exists(uuid): """Check if a Docker container exists for this server.""" name = get_container_name(uuid) rc, out, _ = run_cmd(f'docker ps -a --filter "name=^{name}$" --format "{{{{.Names}}}}"') return rc == 0 and out.strip() == name def docker_server_status(uuid): """Get the status of a Docker container.""" name = get_container_name(uuid) rc, out, _ = run_cmd(f'docker inspect --format "{{{{.State.Status}}}}" {name} 2>/dev/null') if rc == 0: return out.strip() # running, exited, created, paused, etc. return "not_found" def _is_minecraft_server(startup_command, docker_image=""): """Detect if this is a Minecraft server based on startup command or Docker image.""" cmd_lower = startup_command.lower() img_lower = docker_image.lower() # Specific Minecraft keywords (NOT generic Java keywords) mc_keywords = [ "minecraft", "paper", "spigot", "bukkit", "forge", "fabric", "purpur", "bedrock_server", "server.jar", "nogui", "eula", "server.properties" ] mc_images = ["minecraft", "paper", "itzg/"] # Check keywords in command and image keyword_match = ( any(kw in cmd_lower for kw in mc_keywords) or any(kw in img_lower for kw in mc_images) ) # If command contains "java" and ".jar", it COULD be Minecraft, # but only if no non-MC indicators are present if "java" in cmd_lower and ".jar" in cmd_lower: non_mc = ["bungeecord", "velocity", "waterfall", "flamecord", "geyser", "luckperms", "discord", "bot", "proxy"] if not any(kw in cmd_lower or kw in img_lower for kw in non_mc): return True return keyword_match def _is_bukkit_based_server(startup_command, docker_image="", data_dir=None): """ Detect if this is a Bukkit-based server that supports --port CLI flag. (Paper, Spigot, Bukkit, Purpur - NOT Forge, Fabric, BungeeCord, Velocity, Bedrock) """ cmd_lower = startup_command.lower() img_lower = docker_image.lower() # Servers that DON'T support --port non_bukkit = ["forge", "fabric", "bedrock_server", "bungeecord", "velocity", "waterfall", "flamecord", "geyser", "proxy"] if any(kw in cmd_lower or kw in img_lower for kw in non_bukkit): return False # Servers that DO support --port (Bukkit API) bukkit_keywords = ["paper", "spigot", "bukkit", "purpur"] if any(kw in cmd_lower or kw in img_lower for kw in bukkit_keywords): return True # If data_dir provided, check for paper/spigot artifacts if data_dir: data_path = Path(data_dir) if (data_path / "paper.yml").exists() or (data_path / "spigot.yml").exists(): return True # Check jar name for jar in data_path.glob("*.jar"): jar_lower = jar.name.lower() if any(kw in jar_lower for kw in ["paper", "spigot", "purpur"]): return True return False def _patch_server_port(data_dir, port, startup_command="", docker_image=""): """ Patch known game configuration files to use the correct port. Only creates Minecraft-specific files (server.properties, eula.txt) if the server is actually a Minecraft server. """ port = int(port) is_mc = _is_minecraft_server(startup_command, docker_image) # Minecraft: server.properties sp = Path(data_dir) / "server.properties" if sp.exists(): try: content = sp.read_text() # Detect corrupted server.properties (e.g. HTML from a 404 download) # Valid properties files have key=value lines, not HTML tags is_corrupt = ( " 1 # Check TIME_WAIT state rc_tw, out_tw, _ = run_cmd(f"ss -tnp state time-wait sport = :{port}", timeout=5) tw_lines = [l for l in (out_tw or "").strip().splitlines() if l.strip()] if rc_tw == 0 else [] has_time_wait = len(tw_lines) > 1 if not has_listen and not has_time_wait: return None # Port is free # Try to identify which container is using it process_info = listen_lines[1] if has_listen else (tw_lines[1] if has_time_wait else "") state = "LISTEN" if has_listen else "TIME_WAIT" # Try docker: find container using this port rc2, out2, _ = run_cmd( f"docker ps --format '{{{{.Names}}}}' --filter 'network=host' 2>/dev/null", timeout=5, ) return { "port": port, "in_use": True, "state": state, "process": process_info.strip(), "containers": (out2 or "").strip().splitlines() if rc2 == 0 else [], } def _fix_shell_compatibility(data_dir, docker_image=None): """ Fix shell compatibility issues in scripts within data_dir. - Replace #!/bin/bash shebangs with #!/bin/sh (for Alpine/BusyBox images) - Ensure .sh files are executable - Handles the common 'not found' error when bash isn't available """ data_path = Path(data_dir) if not data_path.exists(): return # Check if bash is available in the runtime image has_bash = True if docker_image: rc, _, _ = run_cmd( f"docker run --rm --entrypoint '' {docker_image} test -x /bin/bash 2>/dev/null", timeout=15, ) has_bash = (rc == 0) if has_bash: logger.info(f"Image {docker_image} has /bin/bash - no shebang fixes needed") # Still ensure scripts are executable for sh_file in data_path.glob("*.sh"): try: sh_file.chmod(0o755) except Exception: pass return logger.info(f"Image {docker_image or '(unknown)'} lacks /bin/bash - fixing shebangs") # Fix .sh files in the data directory (top‑level where startup scripts live) for sh_file in data_path.glob("*.sh"): try: content = sh_file.read_text() first_line = content.split('\n')[0] if content else '' if '/bin/bash' in first_line: content = content.replace('#!/bin/bash', '#!/bin/sh', 1) content = content.replace('#!/usr/bin/env bash', '#!/usr/bin/env sh', 1) sh_file.write_text(content) logger.info(f"Fixed shebang in {sh_file.name} → #!/bin/sh") sh_file.chmod(0o755) except Exception as e: logger.warning(f"Failed to fix {sh_file.name}: {e}") def _extract_binary_from_command(cmd): """ Extract the primary executable path from a startup command. Handles exports, pipes, fallbacks (||), and redirections. Returns e.g. './game/bin/linuxsteamrt64/cs2' or 'java' or None. """ if not cmd: return None # Split on && to skip export/env prefix statements parts = [p.strip() for p in cmd.split('&&')] core = parts[-1] # actual command is usually last after exports # Handle || alternatives - take the first one if ' || ' in core: core = core.split(' || ')[0].strip() # Remove redirections (2>/dev/null, >/path, etc.) core = re.sub(r'\d*>[&]?\S+', '', core).strip() # Tokenize tokens = core.split() if not tokens: return None # Skip shell prefixes / wrappers skip = {'exec', 'env', 'nice', 'nohup'} idx = 0 while idx < len(tokens) and tokens[idx] in skip: idx += 1 # 'cd' takes an argument - skip both if idx < len(tokens) and tokens[idx] == 'cd': idx += 2 if idx >= len(tokens): return None return tokens[idx] def _extract_java_major_from_image(docker_image): """Best-effort extraction of Java major version from a Docker image tag.""" if not docker_image: return None img_lower = docker_image.lower() # Pattern 1: explicit "java_NN" or "java:NN" or "java-NN" (e.g. yolks:java_8) match = re.search(r'java[_:-]?(\d+)', img_lower) if match: return int(match.group(1)) # Pattern 2: eclipse-temurin:NN, openjdk:NN, amazoncorretto:NN, etc. # These images use the Java major version as the primary tag component for prefix in ('eclipse-temurin:', 'openjdk:', 'amazoncorretto:', 'adoptopenjdk:', 'ibm-semeru-runtimes:'): if prefix in img_lower: after = img_lower.split(prefix, 1)[1] ver_match = re.match(r'(\d+)', after) if ver_match: return int(ver_match.group(1)) return None def _extract_jar_from_command(cmd): """Extract the JAR path from a java -jar startup command, if present.""" if not cmd: return None match = re.search(r'-jar\s+([^\s]+\.jar)', cmd) if match: return match.group(1).strip('"\'') return None def _class_major_to_java_version(class_major): """Convert a Java class file major version to a Java runtime version.""" if not class_major or class_major < 45: return None return class_major - 44 def _detect_jar_java_requirement(data_dir, startup_command): """Inspect the installed JAR and return its minimum required Java version. Returns (jar_name, class_major, java_major) or (None, None, None). """ data_path = Path(data_dir) if not data_path.exists(): return None, None, None jar_ref = _extract_jar_from_command(startup_command) jar_candidates = [] if jar_ref: jar_candidates.append(data_path / jar_ref.lstrip('./')) for pattern in ("server.jar", "paper*.jar", "spigot*.jar", "forge*.jar", "fabric*.jar", "*.jar"): for jar_path in data_path.glob(pattern): if jar_path not in jar_candidates: jar_candidates.append(jar_path) for jar_path in jar_candidates: if not jar_path.is_file(): continue try: with zipfile.ZipFile(jar_path) as jar_file: class_names = [ name for name in jar_file.namelist() if name.endswith('.class') and not name.startswith('META-INF/') ] if not class_names: continue with jar_file.open(class_names[0]) as class_file: header = class_file.read(8) if len(header) < 8 or header[:4] != b'\xca\xfe\xba\xbe': continue class_major = int.from_bytes(header[6:8], 'big') java_major = _class_major_to_java_version(class_major) if java_major: return jar_path.name, class_major, java_major except Exception as e: logger.warning(f"Failed to inspect JAR {jar_path.name} for Java version: {e}") return None, None, None def _validate_java_runtime_compatibility(data_dir, startup_command, docker_image): """Return an error message when the installed JAR needs a newer Java than the image provides.""" if not _is_minecraft_server(startup_command, docker_image): return None runtime_java = _extract_java_major_from_image(docker_image) if not runtime_java: return None jar_name, class_major, required_java = _detect_jar_java_requirement(data_dir, startup_command) if not required_java or runtime_java >= required_java: return None suggested_image = docker_image if re.search(r'java[_:-]?\d+', docker_image.lower()): suggested_image = re.sub(r'java[_:-]?\d+', f'java_{required_java}', docker_image, flags=re.IGNORECASE) return ( f"Installed JAR {jar_name or 'unknown'} requires Java {required_java} " f"(class file version {class_major}), but runtime image {docker_image} provides Java {runtime_java}. " f"Use {suggested_image} or pin an older Minecraft version." ) # --- Smart Startup Wrapper Template ------------------------------ # Shell script body for the intelligent startup wrapper. # Uses placeholder variables set in the header: STARTUP_CMD, ORIG_BINARY, # IS_FILE_BINARY, BINARY_BASENAME. _SMART_WRAPPER_BODY = r""" log() { echo "[IKABYTE:$1] $2"; } # ═══════════════════════════════════════════════════ # PHASE 1: ENVIRONMENT INFO # ═══════════════════════════════════════════════════ log "BOOT" "IkaByte Smart Startup v3.2.0" log "ENV" "OS=$(uname -s 2>/dev/null) ARCH=$(uname -m 2>/dev/null) KERNEL=$(uname -r 2>/dev/null)" log "ENV" "User=$(id -u 2>/dev/null):$(id -g 2>/dev/null) PWD=$(pwd)" if command -v ldd >/dev/null 2>&1; then LIBC_INFO=$(ldd --version 2>&1 | head -1) log "ENV" "libc: $LIBC_INFO" fi if command -v java >/dev/null 2>&1; then case "$STARTUP_CMD" in *"java "*|*".jar"*) JAVA_INFO=$(java -version 2>&1 | head -1) log "ENV" "java: $JAVA_INFO" ;; esac fi # ═══════════════════════════════════════════════════ # PHASE 2: SHELL COMPATIBILITY # ═══════════════════════════════════════════════════ if ! command -v bash >/dev/null 2>&1; then log "FIX" "bash not available - patching shebangs to /bin/sh" for f in /home/container/*.sh; do [ -f "$f" ] && sed -i '1s|^#!.*/bash.*|#!/bin/sh|' "$f" 2>/dev/null done fi for f in /home/container/*.sh /home/container/*.x86_64 /home/container/bedrock_server /home/container/TShock.Server /home/container/RustDedicated; do [ -f "$f" ] && chmod +x "$f" 2>/dev/null done cd /home/container # ═══════════════════════════════════════════════════ # PHASE 2b: GAME-SPECIFIC PRESTART CHECKS # ═══════════════════════════════════════════════════ # Node.js: auto-install dependencies if package.json exists & node_modules missing/outdated if command -v node >/dev/null 2>&1 && [ -f /home/container/package.json ]; then NEED_INSTALL=0 if [ ! -d /home/container/node_modules ]; then log "FIX" "package.json found but node_modules missing - running npm install" NEED_INSTALL=1 fi if [ "$NEED_INSTALL" = "1" ]; then if command -v npm >/dev/null 2>&1; then cd /home/container npm install --omit=dev 2>&1 | while IFS= read -r line; do log "NPM" "$line"; done # Re-check if node_modules was created if [ -d /home/container/node_modules ]; then log "OK" "npm install completed successfully" else log "WARN" "npm install may have failed - node_modules still missing" fi else log "WARN" "npm not found - cannot install Node.js dependencies" fi else log "OK" "Node.js deps OK (node_modules present)" fi fi # FiveM: license key is mandatory if echo "$STARTUP_CMD" | grep -q "sv_licenseKey" 2>/dev/null; then LICENSE=$(echo "$STARTUP_CMD" | grep -oP 'sv_licenseKey\s+\K\S+' 2>/dev/null || true) if [ -z "$LICENSE" ] || [ "$LICENSE" = '""' ] || [ "$LICENSE" = "''" ]; then log "FATAL" "FiveM license key is NOT SET!" log "FATAL" "Get a free key from: https://keymaster.fivem.net" log "FATAL" "Then set the FIVEM_LICENSE variable in your server settings." log "FATAL" "Server cannot start without a valid license key." exit 1 fi fi # ═══════════════════════════════════════════════════ # PHASE 3: BINARY VALIDATION # ═══════════════════════════════════════════════════ BINARY_PATH="" if [ "$IS_FILE_BINARY" = "1" ] && [ -n "$ORIG_BINARY" ]; then BINARY_PATH="$ORIG_BINARY" if [ ! -f "$BINARY_PATH" ]; then log "WARN" "Binary not found: $BINARY_PATH" log "FIX" "Searching for '$BINARY_BASENAME'..." FOUND=$(find /home/container -maxdepth 6 -name "$BINARY_BASENAME" -type f 2>/dev/null | head -5) if [ -n "$FOUND" ]; then FIRST=$(echo "$FOUND" | head -1) REL=$(echo "$FIRST" | sed 's|^/home/container/|./|') log "FIX" "Found binary at: $REL" # Escape sed special chars in paths OLD_ESC=$(printf '%s\n' "$ORIG_BINARY" | sed 's|[&/]|\\&|g') NEW_ESC=$(printf '%s\n' "$REL" | sed 's|[&/]|\\&|g') NEW_CMD=$(printf '%s' "$STARTUP_CMD" | sed "s|$OLD_ESC|$NEW_ESC|g") if [ "$NEW_CMD" != "$STARTUP_CMD" ]; then STARTUP_CMD="$NEW_CMD" BINARY_PATH="$REL" log "FIX" "Updated command: $STARTUP_CMD" fi EXTRA_MATCHES=$(echo "$FOUND" | wc -l) if [ "$EXTRA_MATCHES" -gt 1 ]; then log "INFO" "Other matches: $(echo "$FOUND" | tail -n +2 | tr '\n' ' ')" fi else log "WARN" "Binary '$BINARY_BASENAME' not found by name - trying auto-detection..." log "DEBUG" "Top-level listing:" ls -la /home/container/ 2>/dev/null | head -30 # -- AUTO-DETECT: known binary names -- AUTO_BIN="" for KNOWN in srcds_linux srcds_run bedrock_server RustDedicated factorio PalServer.sh FXServer gmod; do AB=$(find /home/container -maxdepth 5 -name "$KNOWN" -type f 2>/dev/null | head -1) if [ -n "$AB" ]; then AUTO_BIN="$AB" log "FIX" "Auto-detect: found known binary '$KNOWN'" break fi done # -- AUTO-DETECT: *.x86_64 files -- if [ -z "$AUTO_BIN" ]; then AB=$(find /home/container -maxdepth 5 -name "*.x86_64" -type f 2>/dev/null | head -1) [ -n "$AB" ] && AUTO_BIN="$AB" && log "FIX" "Auto-detect: found .x86_64 binary" fi # -- AUTO-DETECT: start scripts -- if [ -z "$AUTO_BIN" ]; then for SCRIPT in start.sh start_server.sh launch.sh run.sh; do AB=$(find /home/container -maxdepth 3 -name "$SCRIPT" -type f 2>/dev/null | head -1) if [ -n "$AB" ]; then AUTO_BIN="$AB" log "FIX" "Auto-detect: found start script '$SCRIPT'" break fi done fi # -- AUTO-DETECT: largest ELF executable -- if [ -z "$AUTO_BIN" ]; then AUTO_BIN=$(find /home/container -maxdepth 5 -type f -executable \ ! -name "*.txt" ! -name "*.cfg" ! -name "*.json" ! -name "*.yml" \ ! -name "*.xml" ! -name "*.log" ! -name "*.so" ! -name "*.so.*" \ ! -name "*.acf" ! -name "*.vdf" ! -name "*.sh" \ -exec sh -c 'dd if="$1" bs=4 count=1 2>/dev/null | od -A n -t x1 | tr -d " " | grep -q "^7f454c46"' _ {} \; \ -printf '%s %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) [ -n "$AUTO_BIN" ] && log "FIX" "Auto-detect: largest ELF binary -> $(basename "$AUTO_BIN")" fi if [ -n "$AUTO_BIN" ]; then REL=$(echo "$AUTO_BIN" | sed 's|^/home/container/|./|') log "FIX" "Auto-detected binary: $REL" chmod +x "$AUTO_BIN" 2>/dev/null OLD_ESC=$(printf '%s\n' "$ORIG_BINARY" | sed 's|[&/]|\\&|g') NEW_ESC=$(printf '%s\n' "$REL" | sed 's|[&/]|\\&|g') NEW_CMD=$(printf '%s' "$STARTUP_CMD" | sed "s|$OLD_ESC|$NEW_ESC|g") if [ "$NEW_CMD" != "$STARTUP_CMD" ]; then STARTUP_CMD="$NEW_CMD" BINARY_PATH="$REL" log "FIX" "Updated command: $STARTUP_CMD" else STARTUP_CMD="$REL" BINARY_PATH="$REL" log "FIX" "Using auto-detected binary as command: $STARTUP_CMD" fi else PARENT_DIR=$(dirname "$ORIG_BINARY" | sed 's|^\./||') if [ "$PARENT_DIR" != "." ]; then if [ -d "/home/container/$PARENT_DIR" ]; then log "DEBUG" "Contents of $PARENT_DIR/:" ls -la "/home/container/$PARENT_DIR" 2>/dev/null | head -20 fi fi if [ -d "/home/container/steamapps" ]; then MANIFESTS=$(ls /home/container/steamapps/appmanifest_*.acf 2>/dev/null) if [ -n "$MANIFESTS" ]; then log "DEBUG" "SteamCMD app manifests found:" for mf in $MANIFESTS; do APP_ID=$(grep '"appid"' "$mf" 2>/dev/null | head -1 | grep -o '[0-9]*') APP_NAME=$(grep '"name"' "$mf" 2>/dev/null | head -1 | sed 's/.*"\(.*\)"/\1/' | sed 's/^[[:space:]]*//') log "DEBUG" " $(basename "$mf"): appid=$APP_ID name=$APP_NAME" done fi fi log "FATAL" "Cannot start server - binary '$BINARY_BASENAME' not found and auto-detection failed." log "FATAL" "Try reinstalling the server from the admin panel." exit 1 fi fi else log "OK" "Binary found: $BINARY_PATH" fi [ -f "$BINARY_PATH" ] && chmod +x "$BINARY_PATH" 2>/dev/null # ═══════════════════════════════════════════════════ # PHASE 4: DEPENDENCY CHECK # ═══════════════════════════════════════════════════ if command -v ldd >/dev/null 2>&1 && [ -f "$BINARY_PATH" ]; then # Only check ELF binaries (skip scripts) MAGIC=$(od -A n -t x1 -N 4 "$BINARY_PATH" 2>/dev/null | tr -d ' ') if [ "$MAGIC" = "7f454c46" ]; then log "DEPS" "Checking libraries for $BINARY_PATH..." LDD_OUT=$(ldd "$BINARY_PATH" 2>&1 || true) MISSING=$(echo "$LDD_OUT" | grep "not found" || true) if [ -n "$MISSING" ]; then log "WARN" "Missing shared libraries:" echo "$MISSING" | while IFS= read -r line; do log "WARN" " $line"; done log "FIX" "Searching for missing libraries..." for lib in $(echo "$MISSING" | awk '{print $1}'); do FOUND_LIB=$(find /home/container -maxdepth 5 -name "$lib" -type f 2>/dev/null | head -1) if [ -n "$FOUND_LIB" ]; then FOUND_DIR=$(dirname "$FOUND_LIB") export LD_LIBRARY_PATH="${FOUND_DIR}:${LD_LIBRARY_PATH}" log "FIX" "Found $lib -> $FOUND_DIR" fi done STILL=$(ldd "$BINARY_PATH" 2>&1 | grep "not found" || true) if [ -n "$STILL" ]; then log "WARN" "Still missing after fix attempt:" echo "$STILL" | while IFS= read -r line; do log "WARN" " $line"; done else log "OK" "All dependencies resolved via LD_LIBRARY_PATH" fi else log "OK" "All library dependencies satisfied" fi else log "INFO" "$BINARY_PATH is a script or static binary (skipping ldd)" fi fi fi # ═══════════════════════════════════════════════════ # PHASE 5: LIBRARY PATH AUTO-DETECTION # ═══════════════════════════════════════════════════ # Static known paths for libdir in \ /home/container/linux64 \ /home/container/bin/linux64 \ /home/container/game/bin/linuxsteamrt64 \ /home/container/game/bin/linux64 \ /home/container/game/csgo/bin/linuxsteamrt64 \ /home/container/steamcmd/linux64 \ /home/container/steamcmd/linux32 \ /home/container/.steam/sdk64 \ /home/container/.steam/sdk32 \ /home/container/Valheim_server_Data/Plugins/x86_64 \ ; do if [ -d "$libdir" ]; then case "$LD_LIBRARY_PATH" in *"$libdir"*) ;; *) export LD_LIBRARY_PATH="${libdir}:${LD_LIBRARY_PATH}" ;; esac fi done # Dynamic: scan for directories containing .so files (covers ANY game) for libdir in $(find /home/container -maxdepth 4 \( -name "*.so" -o -name "*.so.*" \) -type f 2>/dev/null \ | xargs -I{} dirname {} 2>/dev/null | sort -u 2>/dev/null | head -30); do case "$LD_LIBRARY_PATH" in *"$libdir"*) ;; *) export LD_LIBRARY_PATH="${libdir}:${LD_LIBRARY_PATH}" ;; esac done # Add the binary's own directory (common pattern for game servers) if [ -n "$BINARY_PATH" ] && [ -f "$BINARY_PATH" ]; then BIN_DIR=$(cd "$(dirname "$BINARY_PATH")" 2>/dev/null && pwd || dirname "$BINARY_PATH") case "$LD_LIBRARY_PATH" in *"$BIN_DIR"*) ;; *) export LD_LIBRARY_PATH="${BIN_DIR}:${LD_LIBRARY_PATH}" ;; esac fi [ -n "$LD_LIBRARY_PATH" ] && log "ENV" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" # ═══════════════════════════════════════════════════ # PHASE 6: WAIT FOR PORT & START SERVER # ═══════════════════════════════════════════════════ # Wait for the server port to be free before starting # (prevents 'Address already in use' on restart/crash recovery) if [ -n "$SERVER_PORT" ]; then PORT_WAIT=0 PORT_MAX_WAIT=90 while [ "$PORT_WAIT" -lt "$PORT_MAX_WAIT" ]; do PORT_FREE=1 if command -v ss >/dev/null 2>&1; then PORT_CHECK=$(ss -tlnH sport = :${SERVER_PORT} 2>/dev/null | head -1) TW_CHECK=$(ss -tnH state time-wait sport = :${SERVER_PORT} 2>/dev/null | head -1) [ -n "$PORT_CHECK" ] || [ -n "$TW_CHECK" ] && PORT_FREE=0 elif command -v netstat >/dev/null 2>&1; then PORT_CHECK=$(netstat -tln 2>/dev/null | grep ":${SERVER_PORT} " | head -1) TW_CHECK=$(netstat -tn 2>/dev/null | grep ":${SERVER_PORT} .*TIME_WAIT" | head -1) [ -n "$PORT_CHECK" ] || [ -n "$TW_CHECK" ] && PORT_FREE=0 elif command -v python3 >/dev/null 2>&1; then python3 -c " import socket,sys s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) try: s.bind(('0.0.0.0',${SERVER_PORT})); s.close() except: sys.exit(1) " 2>/dev/null || PORT_FREE=0 elif [ -f /proc/net/tcp ]; then # Universal fallback: parse /proc/net/tcp (works on ALL Linux without extra tools) # Port is in hex at column 2 (local_address), states: 0A=LISTEN, 06=TIME_WAIT PORT_HEX=$(printf '%04X' ${SERVER_PORT}) if grep -qi ":${PORT_HEX} " /proc/net/tcp 2>/dev/null; then PORT_FREE=0 fi else break # No tool available, skip check fi if [ "$PORT_FREE" -eq 1 ]; then break # Port is fully free fi if [ "$PORT_WAIT" -eq 0 ]; then log "WAIT" "Port $SERVER_PORT busy (LISTEN or TIME_WAIT), waiting up to ${PORT_MAX_WAIT}s..." fi sleep 1 PORT_WAIT=$((PORT_WAIT + 1)) done if [ "$PORT_WAIT" -ge "$PORT_MAX_WAIT" ]; then log "WARN" "Port $SERVER_PORT still busy after ${PORT_MAX_WAIT}s - starting anyway" elif [ "$PORT_WAIT" -gt 0 ]; then log "START" "Port $SERVER_PORT free after ${PORT_WAIT}s wait" fi fi # Log only the actual server command (strip web blocker prefix if present) CLEAN_CMD=$(echo "$STARTUP_CMD" | sed 's/^(.*) & //') log "START" "Executing: $CLEAN_CMD" exec /bin/sh -c "$STARTUP_CMD" """ def _build_startup_wrapper(server_dir, data_dir, final_cmd): """ Create an intelligent startup wrapper script. The wrapper validates the environment, resolves binary paths, checks shared library dependencies, auto-fixes LD_LIBRARY_PATH, and logs structured diagnostic information - all INSIDE the container. Returns (container_cmd, extra_volume_flag) tuple. """ wrapper_path = server_dir / ".ikabyte_start.sh" # Extract binary info for inline validation binary = _extract_binary_from_command(final_cmd) is_file_binary = bool(binary and (binary.startswith('./') or binary.startswith('/'))) binary_basename = os.path.basename(binary) if binary else "" # Escape single quotes for safe shell embedding esc = lambda s: s.replace("'", "'\\''") header = ( "#!/bin/sh\n" "# IkaByte Game Agent v3.2.0 - Smart Startup Wrapper\n" "# Auto-generated - do not edit. Mounted read-only.\n\n" f"STARTUP_CMD='{esc(final_cmd)}'\n" f"ORIG_BINARY='{esc(binary or '')}'\n" f"IS_FILE_BINARY={'1' if is_file_binary else '0'}\n" f"BINARY_BASENAME='{esc(binary_basename)}'\n" ) wrapper_content = header + _SMART_WRAPPER_BODY try: with open(wrapper_path, "w", newline="\n") as f: f.write(wrapper_content) wrapper_path.chmod(0o755) run_cmd(f"chown root:root {wrapper_path}") logger.info(f"Created smart startup wrapper: {wrapper_path}") except Exception as e: logger.warning(f"Failed to create startup wrapper: {e}") return final_cmd, "" # fallback to raw command # Mount wrapper at /.ikabyte_start.sh (root of container, read-only) volume_flag = f" -v {wrapper_path}:/.ikabyte_start.sh:ro" return "/bin/sh /.ikabyte_start.sh", volume_flag def _post_install_validate(data_dir, startup_command): """ Quick host-side validation after install script completes. Checks that expected binary exists and data dir is not empty. Returns list of issue strings (empty = all good). """ binary = _extract_binary_from_command(startup_command) issues = [] if binary and (binary.startswith('./') or binary.startswith('/')): bin_rel = binary[2:] if binary.startswith('./') else binary.lstrip('/') bin_path = Path(data_dir) / bin_rel if not bin_path.exists(): issues.append(f"Startup binary not found at expected path: {binary}") parent = bin_path.parent if parent.exists(): siblings = [f.name for f in parent.iterdir()][:20] logger.warning( f"Post-install: binary {binary} not found. " f"Parent dir ({parent.name}/) contains: {siblings}" ) else: logger.warning(f"Post-install: binary parent dir {parent} doesn't exist") else: # Ensure executable if not os.access(bin_path, os.X_OK): try: bin_path.chmod(0o755) logger.info(f"Post-install: fixed permissions on {binary}") except Exception: pass logger.info(f"Post-install: binary found at {bin_path}") # Check data dir is not empty try: has_files = any(Path(data_dir).iterdir()) except Exception: has_files = False if not has_files: issues.append("Data directory is empty after install") logger.warning("Post-install: data directory is empty!") if not issues: logger.info("Post-install validation: OK") return issues def _post_start_health_check(uuid, name, wait_seconds=8): """ Check if a server container is still running shortly after start. If it crashed, analyze logs for common error patterns and return diagnosis. """ time.sleep(wait_seconds) status = docker_server_status(uuid) if status == "running": logger.info(f"Health check: {name} is running (healthy)") return {"healthy": True, "status": status} # Container exited - analyze logs rc, logs, _ = run_cmd(f"docker logs --tail 120 {name} 2>&1", timeout=10) log_text = logs if rc == 0 else "" diagnosis = { "healthy": False, "status": status, "exit_code": None, "error_type": "unknown", "error_detail": "", "suggestion": "", "logs_tail": log_text[-2000:] if log_text else "", } # Get exit code rc2, exit_out, _ = run_cmd( f'docker inspect --format "{{{{.State.ExitCode}}}}" {name} 2>/dev/null', timeout=5 ) if rc2 == 0: diagnosis["exit_code"] = exit_out.strip().strip("'\"") # Pattern matching on logs for common errors lt = log_text.lower() if "not found" in lt: if "bin/bash" in log_text: diagnosis["error_type"] = "missing_bash" diagnosis["error_detail"] = "Container image does not have /bin/bash" diagnosis["suggestion"] = "Image should use /bin/sh or install bash" elif any(x in log_text for x in ['.so', 'libsteam', 'libSDL', 'libgcc']): diagnosis["error_type"] = "missing_library" diagnosis["error_detail"] = "Required shared library (.so) not found" diagnosis["suggestion"] = "Check LD_LIBRARY_PATH - wrapper should auto-detect" else: diagnosis["error_type"] = "binary_not_found" diagnosis["error_detail"] = "Startup binary not found at expected path" diagnosis["suggestion"] = "Check install script output and binary path in egg config" elif "permission denied" in lt: diagnosis["error_type"] = "permission_denied" diagnosis["error_detail"] = "Binary or script lacks execute permission" diagnosis["suggestion"] = "Wrapper should auto-fix - check file ownership" elif "address already in use" in lt or "bind failed" in lt: diagnosis["error_type"] = "port_conflict" diagnosis["error_detail"] = "Port is already in use by another process" diagnosis["suggestion"] = "Stop the conflicting service or change the server port" elif "eula" in lt and "agree" in lt: diagnosis["error_type"] = "eula_not_accepted" diagnosis["error_detail"] = "Minecraft EULA not accepted" diagnosis["suggestion"] = "eula.txt should be auto-created - check agent port patching" elif ("java" in lt or "jvm" in lt) and ("error" in lt or "exception" in lt): diagnosis["error_type"] = "java_error" diagnosis["error_detail"] = "Java runtime error or exception" diagnosis["suggestion"] = "Check Java version and memory settings" elif "segfault" in lt or "segmentation fault" in lt: diagnosis["error_type"] = "segfault" diagnosis["error_detail"] = "Server binary crashed (segmentation fault)" diagnosis["suggestion"] = "May need different Docker image or missing libraries" elif "sigabrt" in lt or "signal: aborted" in lt: # Check for known SIGABRT causes if "license" in lt and ("key" in lt or "specified" in lt): diagnosis["error_type"] = "fivem_no_license" diagnosis["error_detail"] = "FiveM license key is missing or empty" diagnosis["suggestion"] = "Set FIVEM_LICENSE variable - get a key from https://keymaster.fivem.net" else: diagnosis["error_type"] = "sigabrt" diagnosis["error_detail"] = "Server crashed with SIGABRT (abort signal)" diagnosis["suggestion"] = "Check if the Docker image has glibc (not musl/Alpine) and all required config is set" elif "killed" in lt or diagnosis.get("exit_code") == "137": diagnosis["error_type"] = "oom_killed" diagnosis["error_detail"] = "Process was killed (possibly out of memory)" diagnosis["suggestion"] = "Increase memory limit for this server" logger.warning( f"Health check FAILED for {name}: status={status}, " f"exit_code={diagnosis['exit_code']}, type={diagnosis['error_type']}, " f"detail={diagnosis['error_detail']}" ) return diagnosis # --- Install status tracking ------------------------------------- def _write_install_status(uuid, status, message="", **extra): """Write install progress to a JSON file the panel can poll.""" server_dir = Path(DATA_DIR) / "servers" / uuid server_dir.mkdir(parents=True, exist_ok=True) status_file = server_dir / "install_status.json" payload = { "status": status, # queued, pulling, installing, configuring, starting, done, failed "message": message, "updated_at": time.strftime("%Y-%m-%dT%H:%M:%S"), **extra, } try: with open(status_file, "w") as f: json.dump(payload, f, indent=2) except Exception as e: logger.warning(f"Failed to write install status for {uuid}: {e}") def _read_install_status(uuid): """Read current install status for a server.""" status_file = Path(DATA_DIR) / "servers" / uuid / "install_status.json" if not status_file.exists(): return {"status": "unknown", "message": "No install status found"} try: with open(status_file) as f: return json.load(f) except Exception: return {"status": "unknown", "message": "Failed to read install status"} # --- LinuxGSM support -------------------------------------------- # Track active LinuxGSM message injector threads {uuid: threading.Event} _lgsm_injectors = {} def _is_linuxgsm_server(startup_command, environment): """Check if a server should use LinuxGSM mode.""" if startup_command == "__LINUXGSM__": return True if "LGSM_GAMESERVER" in environment: return True return False def _is_linuxgsm_container(uuid): """Check if an installed server uses LinuxGSM (from saved config).""" config_file = Path(DATA_DIR) / "servers" / uuid / "config.json" if config_file.exists(): try: with open(config_file) as f: cfg = json.load(f) return _is_linuxgsm_server( cfg.get("startup_command", ""), cfg.get("environment", {}), ) except Exception: pass return False def _start_linuxgsm_message_injector(uuid, name): """Start a background thread that periodically writes an IKABYTE banner to the container's main stdout (visible in docker logs / console).""" # Stop any existing injector for this server if uuid in _lgsm_injectors: _lgsm_injectors[uuid].set() stop_event = threading.Event() _lgsm_injectors[uuid] = stop_event def _injector(): msg = ( "\\n═══════════════════════════════════════════════════════════════\\n" " IKABYTE MANAGER: Ten serwer dziala na LinuxGSM\\n" " Zarzadzany automatycznie przez IkaByte Game Agent v{ver}\\n" "═══════════════════════════════════════════════════════════════\\n" ).format(ver=AGENT_VERSION) while not stop_event.is_set(): stop_event.wait(300) # every 5 minutes if stop_event.is_set(): break status = docker_server_status(uuid) if status != "running": break run_cmd( f"docker exec --user 0 {name} sh -c 'echo -e \"{msg}\" > /proc/1/fd/1'", timeout=10, ) _lgsm_injectors.pop(uuid, None) thread = threading.Thread(target=_injector, daemon=True, name=f"lgsm-msg-{uuid[:12]}") thread.start() logger.info(f"[{uuid}] LinuxGSM message injector started") def _write_linuxgsm_port_config(uuid, name, safe_lgsm, port, server_dir): """Write LinuxGSM port config using docker cp to avoid shell quoting issues. LinuxGSM expects config values in the format: port="27015" Writing via echo in bash strips the quotes, so we write a temp file on the host and docker cp it into the container. """ cfg_path_in_container = f"/data/config-lgsm/{safe_lgsm}/{safe_lgsm}.cfg" tmp_cfg = server_dir / "lgsm_port.cfg" try: # Write config file on host with correct format (quotes preserved) with open(tmp_cfg, "w", newline="\n") as f: f.write(f'port="{port}"\n') # Ensure config dir exists inside container run_cmd( f"docker exec {name} mkdir -p /data/config-lgsm/{safe_lgsm}", timeout=10, ) # Copy config file into container rc, _, err = run_cmd( f"docker cp {tmp_cfg} {name}:{cfg_path_in_container}", timeout=10, ) if rc == 0: logger.info(f"[{uuid}] LinuxGSM port configured: {port} -> {cfg_path_in_container}") else: logger.warning(f"[{uuid}] Failed to docker cp port config: {err}") except Exception as e: logger.warning(f"[{uuid}] Failed to write LinuxGSM port config: {e}") finally: # Clean up temp file try: tmp_cfg.unlink(missing_ok=True) except Exception: pass def _validate_docker_image(image_name): """Validate Docker image name to prevent command injection.""" if not image_name or not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9._/:-]*$', image_name): return False # Block shell metacharacters if any(c in image_name for c in ';|&$`\\(){}[]!#~'): return False return True def _install_linuxgsm_server(uuid, name, data, server_dir, data_dir): """Install a LinuxGSM-managed game server. LinuxGSM Docker images set up the LinuxGSM environment via their entrypoint but do NOT auto-install game files. After starting the container we must explicitly run ``auto-install`` inside it, then write port configuration to the LinuxGSM config directory so the correct allocated port is used. """ docker_image = data.get("docker_image", "") environment = data.get("environment", {}) memory_limit = data.get("memory_limit", 1024) cpu_limit = data.get("cpu_limit", 100) port = data.get("port") lgsm_gameserver = environment.get("LGSM_GAMESERVER", "") if not lgsm_gameserver: _write_install_status(uuid, "failed", "Missing LGSM_GAMESERVER environment variable - cannot determine game type") return logger.info(f"[{uuid}] LinuxGSM mode - image: {docker_image}, game: {lgsm_gameserver}") # --- Validate + Pull image ---------------------------------- if not _validate_docker_image(docker_image): _write_install_status(uuid, "failed", f"Invalid Docker image name: {docker_image}") return _write_install_status(uuid, "pulling", f"Pulling LinuxGSM image: {docker_image}") rc, _, err = run_cmd(f"docker pull {docker_image}", timeout=600) if rc != 0: _write_install_status(uuid, "failed", f"Failed to pull LinuxGSM image: {err}") return # --- Build env flags ---------------------------------------- env_flags = "" for key, value in environment.items(): safe_key = re.sub(r'[^A-Za-z0-9_]', '', str(key)) safe_val = str(value).replace("'", "'\\''") env_flags += f" -e '{safe_key}={safe_val}'" mem_bytes = int(memory_limit) * 1024 * 1024 cpu_shares = max(2, int(int(cpu_limit) * 1024 / 100)) # Remove old container run_cmd(f"docker rm -f {name} 2>/dev/null", timeout=15) # --- Create container (no CMD/user override - LinuxGSM entrypoint) -- _write_install_status(uuid, "configuring", "Creating LinuxGSM container...") docker_cmd = ( f"docker create" f" --name {name}" f" --network host" f" --memory={mem_bytes}" f" --cpu-shares={cpu_shares}" f" --pids-limit=512" f" --restart=no" f" -v {data_dir}:/data" f" {env_flags}" f" -e HOME=/data" f" -e 'PS1=container@ikabyte~ '" f" -e SERVER_PORT={port or _get_default_port(data.get('startup_command', ''), docker_image, environment)}" f" -e SERVER_MEMORY={memory_limit}" f" --tty --interactive" f" {docker_image}" ) logger.info(f"[{uuid}] Creating LinuxGSM container: {name}") rc, out, err = run_cmd(docker_cmd, timeout=60) if rc != 0: _write_install_status(uuid, "failed", f"Failed to create LinuxGSM container: {err}") return # --- Start container (entrypoint initialises LinuxGSM) ------ _write_install_status(uuid, "configuring", "Starting container - LinuxGSM initialising...") rc, _, err = run_cmd(f"docker start {name}", timeout=30) if rc != 0: _write_install_status(uuid, "failed", f"Failed to start LinuxGSM container: {err}") return # Wait for LinuxGSM entrypoint to finish initialising (creates scripts, dirs, npm deps) logger.info(f"[{uuid}] Waiting for LinuxGSM entrypoint to initialise...") time.sleep(20) # Verify container is still running after init status = docker_server_status(uuid) if status != "running": _write_install_status(uuid, "failed", f"LinuxGSM container exited during initialisation (status: {status})") return # --- Write port configuration ------------------------------- safe_lgsm = re.sub(r'[^a-zA-Z0-9_-]', '', lgsm_gameserver) if port: _write_linuxgsm_port_config(uuid, name, safe_lgsm, port, server_dir) # --- Run game installation via auto-install ----------------- _write_install_status(uuid, "installing", f"Installing game files via LinuxGSM ({lgsm_gameserver}) - this may take 30-60+ minutes for large games...") logger.info(f"[{uuid}] Running LinuxGSM auto-install for {lgsm_gameserver}...") rc_inst, out_inst, err_inst = run_cmd( f"docker exec {name} /app/{lgsm_gameserver} auto-install", timeout=7200, # 2 hours - large games like CS2/ARK need time ) install_log = (out_inst or "") + ("\n--- STDERR ---\n" + err_inst if err_inst else "") logger.info(f"[{uuid}] LinuxGSM auto-install output (last 2000 chars): {install_log[-2000:]}") if rc_inst != 0: logger.error(f"[{uuid}] LinuxGSM auto-install FAILED (rc={rc_inst})") _write_install_status(uuid, "failed", f"LinuxGSM auto-install failed with exit code {rc_inst}", exit_code=rc_inst, install_log=install_log[-3000:]) return # Re-write port config after auto-install (auto-install may create default configs) if port: _write_linuxgsm_port_config(uuid, name, safe_lgsm, port, server_dir) # --- Start the game server ---------------------------------- _write_install_status(uuid, "starting", "Starting game server...") logger.info(f"[{uuid}] Starting {lgsm_gameserver} via LinuxGSM...") rc_start, out_start, err_start = run_cmd( f"docker exec {name} /app/{lgsm_gameserver} start", timeout=120, ) if rc_start != 0: logger.warning(f"[{uuid}] LinuxGSM start returned rc={rc_start}: {err_start}") # Re-write port config AFTER game start (game servers may reset to defaults) if port: time.sleep(5) _write_linuxgsm_port_config(uuid, name, safe_lgsm, port, server_dir) # Start the periodic console message injector _start_linuxgsm_message_injector(uuid, name) # Give the game server a moment to spin up, then check time.sleep(10) status = docker_server_status(uuid) if status == "running": _write_install_status(uuid, "done", f"LinuxGSM {lgsm_gameserver} installed and started successfully", healthy=True, container=name, linuxgsm=True) logger.info(f"[{uuid}] LinuxGSM install complete - server running") else: _write_install_status(uuid, "done", f"LinuxGSM {lgsm_gameserver} installed - container status: {status}", healthy=False, container=name, linuxgsm=True) logger.warning(f"[{uuid}] LinuxGSM container status after game start: {status}") # --- Auto-Detection & SteamCMD Fallback -------------------------- def _is_elf_binary(filepath): """Check if a file is an ELF binary by reading its magic bytes.""" try: with open(filepath, 'rb') as f: return f.read(4) == b'\x7fELF' except Exception: return False def _auto_detect_binary(data_dir): """Automatically detect the server binary in the data directory. Scans using a priority system: 1. Known server binary names (srcds_linux, bedrock_server, etc.) 2. Files matching *.x86_64, *.x86 3. Startup scripts (start.sh, run.sh, launch.sh) 4. .jar files (Java servers like Minecraft) 5. Largest ELF executable file 6. First executable found (last resort) Returns (relative_path: str | None, confidence: float 0.0-1.0). """ data_path = Path(data_dir) if not data_path.exists(): return None, 0.0 # Priority 1: Known binary names known_binaries = [ "srcds_linux", "srcds_run", "bedrock_server", "PalServer.sh", "RustDedicated", "TShock.Server", "factorio", "valheim_server.x86_64", "7DaysToDieServer.x86_64", "ArkAscendedServer.sh", "ShooterGameServer", "FXServer", "FXServer.exe", "PaperMC.jar", "paper.jar", "server.jar", "spigot.jar", "fabric-server-launch.jar", "gmod", "garrysmod", "srcds", ] for bname in known_binaries: matches = list(data_path.rglob(bname)) if matches: rel = str(matches[0].relative_to(data_path)) logger.info(f"Auto-detect binary: found known binary '{bname}' at {rel}") return f"./{rel}", 0.95 # Priority 2: *.x86_64, *.x86 for pattern in ("*.x86_64", "*.x86"): matches = [f for f in data_path.rglob(pattern) if f.is_file()] if matches: largest = max(matches, key=lambda f: f.stat().st_size) rel = str(largest.relative_to(data_path)) return f"./{rel}", 0.85 # Priority 3: Start scripts for script_name in ("start.sh", "start_server.sh", "launch.sh", "run.sh"): matches = list(data_path.rglob(script_name)) if matches: rel = str(matches[0].relative_to(data_path)) return f"./{rel}", 0.80 # Priority 4: .jar files (Java game servers) jar_matches = [f for f in data_path.rglob("*.jar") if f.is_file()] if jar_matches: # Prefer server.jar, paper*.jar, etc. for jar in jar_matches: if any(kw in jar.name.lower() for kw in ("server", "paper", "spigot", "forge", "fabric")): rel = str(jar.relative_to(data_path)) return f"./{rel}", 0.80 largest_jar = max(jar_matches, key=lambda f: f.stat().st_size) rel = str(largest_jar.relative_to(data_path)) return f"./{rel}", 0.60 # Priority 5: Largest ELF executable skip_ext = {'.txt', '.cfg', '.ini', '.json', '.yml', '.yaml', '.xml', '.md', '.log', '.pid', '.lock', '.acf', '.vdf', '.png', '.jpg', '.gif', '.tar', '.gz', '.zip', '.bz2', '.xz', '.so', '.html', '.css', '.js'} executables = [] try: for f in data_path.rglob("*"): if f.is_file() and f.suffix.lower() not in skip_ext: if _is_elf_binary(f): executables.append(f) except Exception: pass if executables: largest = max(executables, key=lambda f: f.stat().st_size) rel = str(largest.relative_to(data_path)) logger.info(f"Auto-detect binary: largest ELF executable -> {rel} ({largest.stat().st_size} bytes)") return f"./{rel}", 0.50 # Priority 6: Any executable file try: for f in data_path.rglob("*"): if f.is_file() and f.suffix.lower() not in skip_ext and os.access(f, os.X_OK): rel = str(f.relative_to(data_path)) return f"./{rel}", 0.30 except Exception: pass return None, 0.0 def _steamcmd_install(data_dir, app_id, validate=True): """Run SteamCMD inside a Docker container to install/update a game. Phase 1: tries pre-built SteamCMD images (steamcmd/steamcmd, cm2network). Phase 2: falls back to ubuntu:22.04 with SteamCMD installed at runtime. Retries up to 5 times per phase (SteamCMD is notoriously flaky). Returns (success: bool, message: str). """ if not app_id or not str(app_id).strip().isdigit(): return False, f"Invalid Steam App ID: {app_id}" app_id = str(app_id).strip() validate_flag = "validate" if validate else "" game_name = LARGE_STEAM_GAMES.get(app_id, f"app {app_id}") min_free_gb = MIN_STEAM_DISK_GB.get(app_id, 5) free_gb = None logger.info(f"SteamCMD: Installing app {app_id} to {data_dir}") # Check available disk space try: stat = os.statvfs(data_dir) free_gb = (stat.f_frsize * stat.f_bavail) / (1024**3) logger.info(f"SteamCMD: {free_gb:.1f} GB free on {data_dir}") if free_gb < min_free_gb: return False, f"Insufficient disk space for {game_name}: {free_gb:.1f} GB free (need at least {min_free_gb} GB)" except Exception as e: logger.warning(f"Could not check disk space: {e}") # Ensure data dir is writable run_cmd(f"chmod -R 755 {data_dir}") # -- Phase 1: Pre-built SteamCMD image -------------------- steamcmd_image = None for img in ("steamcmd/steamcmd:latest", "cm2network/steamcmd:latest"): rc, _, _ = run_cmd(f"docker pull {img}", timeout=300) if rc == 0: steamcmd_image = img break if steamcmd_image: container_name = f"steamcmd-{app_id}-{int(time.time()) % 100000}" # Both steamcmd/steamcmd and cm2network have ENTRYPOINT set to steamcmd, # so we only pass SteamCMD arguments (NOT 'steamcmd' as CMD). cmd = ( f"docker run --rm --name {container_name}" f" -v {data_dir}:/data" f" --user root" f" {steamcmd_image}" f" +@sSteamCmdForcePlatformType linux" f" +login anonymous" f" +force_install_dir /data" f" +app_update {app_id} {validate_flag}" f" +quit" ) # Determine timeout based on game size timeout_seconds = 10800 if app_id in LARGE_STEAM_GAMES else 7200 for attempt in range(1, 6): logger.info( f"SteamCMD install attempt {attempt}/5 for {game_name} " f"({steamcmd_image}, timeout: {timeout_seconds}s = {timeout_seconds/3600:.1f}h)..." ) rc, out, err = run_cmd(cmd, timeout=timeout_seconds) combined = (out or "") + (err or "") if rc == 0: data_path = Path(data_dir) has_manifest = any(data_path.rglob("appmanifest_*.acf")) has_files = sum(1 for _ in data_path.iterdir()) > 1 if has_manifest or has_files: logger.info(f"SteamCMD: {game_name} installed successfully") return True, "SteamCMD install successful" fatal_reason = _steamcmd_failure_reason(combined, app_id, free_gb) if fatal_reason: return False, fatal_reason if attempt < 5: delay = 15 * attempt # Longer delays for large games logger.warning(f"SteamCMD attempt {attempt} failed (rc={rc}), retrying in {delay}s...") run_cmd(f"docker rm -f {container_name} 2>/dev/null", timeout=10) time.sleep(delay) logger.warning(f"SteamCMD image {steamcmd_image} failed after 5 attempts, trying ubuntu fallback...") # -- Phase 2: Ubuntu fallback - install SteamCMD at runtime - logger.info(f"SteamCMD: Ubuntu fallback for app {app_id}") rc, _, err = run_cmd("docker pull ubuntu:22.04", timeout=300) if rc != 0: return False, f"Failed to pull any SteamCMD-capable image: {err}" script = ( "#!/bin/bash\n" "set -e\n" "export DEBIAN_FRONTEND=noninteractive\n" "dpkg --add-architecture i386\n" "apt-get update -y\n" "apt-get install -y --no-install-recommends lib32gcc-s1 lib32stdc++6 curl ca-certificates\n" "mkdir -p /tmp/steamcmd && cd /tmp/steamcmd\n" '[ -f steamcmd.sh ] || curl -sqL "https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz" | tar zxf -\n' "STEAM_OK=0\n" "for i in 1 2 3 4 5; do\n" f' echo "SteamCMD attempt $i/5 for app {app_id}..."\n' f" /tmp/steamcmd/steamcmd.sh +@sSteamCmdForcePlatformType linux +login anonymous" f" +force_install_dir /data +app_update {app_id} {validate_flag} +quit" " && { STEAM_OK=1; break; }\n" ' echo "Attempt $i failed, waiting 15s..."\n' " sleep 15\n" "done\n" '[ "$STEAM_OK" -eq 1 ] || { echo "ERROR: SteamCMD failed after 5 attempts"; exit 1; }\n' 'echo "SteamCMD install completed successfully"\n' ) script_path = Path(data_dir).parent / "steamcmd_helper.sh" with open(script_path, "w", newline="\n") as f: f.write(script) script_path.chmod(0o755) container_name = f"steamcmd-ub-{app_id}-{int(time.time()) % 100000}" cmd = ( f"docker run --rm --name {container_name}" f" -v {data_dir}:/data" f" -v {script_path}:/tmp/install.sh:ro" f" ubuntu:22.04" f" /bin/bash /tmp/install.sh" ) logger.info(f"SteamCMD ubuntu fallback for app {app_id}...") timeout_seconds = 10800 if app_id in LARGE_STEAM_GAMES else 7200 rc, out, err = run_cmd(cmd, timeout=timeout_seconds) try: script_path.unlink() except Exception: pass combined = (out or "") + (err or "") if rc == 0: data_path = Path(data_dir) has_files = sum(1 for _ in data_path.iterdir()) > 0 if has_files: logger.info(f"SteamCMD: App {app_id} installed successfully (ubuntu fallback)") return True, "SteamCMD install successful" fatal_reason = _steamcmd_failure_reason(combined, app_id, free_gb) if fatal_reason: return False, fatal_reason return False, f"SteamCMD failed after all attempts (last rc={rc})" def _steamcmd_validate_install(data_dir): """Check if a SteamCMD installation looks valid. Returns (is_valid: bool, details: list[dict]). """ data_path = Path(data_dir) steamapps = data_path / "steamapps" if not steamapps.exists(): return False, "No steamapps directory found" manifests = list(steamapps.glob("appmanifest_*.acf")) if not manifests: return False, "No app manifests found in steamapps/" details = [] for mf in manifests: try: content = mf.read_text() app_id_m = re.search(r'"appid"\s+"(\d+)"', content) name_m = re.search(r'"name"\s+"([^"]+)"', content) state_m = re.search(r'"StateFlags"\s+"(\d+)"', content) details.append({ "manifest": mf.name, "app_id": app_id_m.group(1) if app_id_m else "unknown", "name": name_m.group(1) if name_m else "unknown", "state": state_m.group(1) if state_m else "unknown", "fully_installed": state_m is not None and state_m.group(1) == "4", }) except Exception: pass all_installed = all(d.get("fully_installed", False) for d in details) return all_installed, details def _self_healing_check(uuid, name=None): """Attempt to automatically fix common server issues. Checks: binary existence, port config, file permissions, startup command. Returns dict with list of actions taken. """ if name is None: name = get_container_name(uuid) server_dir = Path(DATA_DIR) / "servers" / uuid config_file = server_dir / "config.json" data_dir = server_dir / "data" actions = [] if not config_file.exists(): return {"actions": actions, "error": "No config found"} try: with open(config_file) as f: cfg = json.load(f) except Exception as e: return {"actions": actions, "error": f"Failed to read config: {e}"} startup_command = cfg.get("startup_command", "") environment = cfg.get("environment", {}) port = cfg.get("port") docker_image = cfg.get("docker_image", "") install_script = cfg.get("install_script", "") server_type = ServerTypeDetector.detect(startup_command, environment, install_script, docker_image) # -- Fix 1: Missing binary -------------------------------- binary = _extract_binary_from_command(startup_command) if binary and (binary.startswith('./') or binary.startswith('/')): bin_rel = binary[2:] if binary.startswith('./') else binary.lstrip('/') bin_path = data_dir / bin_rel if not bin_path.exists(): logger.info(f"[{uuid}] Self-heal: binary {binary} missing - auto-detecting...") detected, confidence = _auto_detect_binary(str(data_dir)) if detected: new_cmd = startup_command.replace(binary, detected) cfg["startup_command"] = new_cmd try: with open(config_file, "w") as f: json.dump(cfg, f, indent=2) except Exception: pass actions.append(f"Fixed binary: {binary} -> {detected} (conf={confidence:.0%})") logger.info(f"[{uuid}] Self-heal: updated binary -> {detected}") else: # If SteamCMD server, try re-downloading app_id = None if server_type == ServerTypeDetector.STEAMCMD: app_id = ServerTypeDetector.get_steam_app_id(environment, install_script) if app_id: actions.append(f"Binary missing, running SteamCMD repair (app {app_id})...") success, msg = _steamcmd_install(str(data_dir), app_id) actions.append(f"SteamCMD repair: {'OK' if success else 'FAILED'} - {msg}") if success: run_cmd(f"chown -R 1000:1000 {data_dir}") else: actions.append(f"Binary {binary} missing - no auto-fix available") # -- Fix 2: Port configuration ---------------------------- if port: _patch_server_port(data_dir, port or _get_default_port(startup_command, docker_image, environment), startup_command, docker_image) actions.append(f"Re-applied port config: {port}") # -- Fix 3: Permissions & shell compatibility ------------- if data_dir.exists(): run_cmd(f"chown -R 1000:1000 {data_dir}") _fix_shell_compatibility(data_dir, docker_image) actions.append("Fixed permissions and shell compatibility") return {"actions": actions} def _install_server_sync(data): """Synchronous server install - runs in a background thread.""" uuid = data.get("uuid", "") name = get_container_name(uuid) docker_image = data.get("docker_image", "") startup_command = data.get("startup_command", "") stop_command = data.get("stop_command", "") install_script = data.get("install_script", "") install_docker_image = data.get("install_docker_image", "") memory_limit = data.get("memory_limit", 1024) disk_limit = data.get("disk_limit", 10240) cpu_limit = data.get("cpu_limit", 100) port = data.get("port") environment = data.get("environment", {}) server_name = data.get("name", uuid) server_dir = Path(DATA_DIR) / "servers" / uuid server_dir.mkdir(parents=True, exist_ok=True) data_dir = server_dir / "data" data_dir.mkdir(parents=True, exist_ok=True) try: # Save server config config = { "uuid": uuid, "name": server_name, "docker_image": docker_image, "startup_command": startup_command, "stop_command": stop_command, "install_script": install_script, "install_docker_image": install_docker_image, "memory_limit": memory_limit, "disk_limit": disk_limit, "cpu_limit": cpu_limit, "port": port, "environment": environment, "installed_at": time.strftime("%Y-%m-%d %H:%M:%S"), } config_file = server_dir / "config.json" with open(config_file, "w") as f: json.dump(config, f, indent=2) # --- Security analysis ---------------------------------- if install_script: is_safe, sec_warnings = SecurityManager.analyze_script(install_script) if sec_warnings: logger.warning(f"[{uuid}] Security scan found {len(sec_warnings)} warning(s):") for w in sec_warnings: logger.warning(f" Line {w['line']}: {w['reason']} - {w['content'][:100]}") if not is_safe: _write_install_status(uuid, "failed", "Install script blocked by security analysis - contains dangerous commands", security_warnings=sec_warnings[:10]) return # --- Auto-detect server type ---------------------------- server_type = ServerTypeDetector.detect(startup_command, environment, install_script, docker_image) logger.info(f"[{uuid}] Server type detected: {server_type}") # --- LinuxGSM mode -------------------------------------- if _is_linuxgsm_server(startup_command, environment): _install_linuxgsm_server(uuid, name, data, server_dir, data_dir) return # Ensure container user (uid 1000) can write to data dir run_cmd(f"chown -R 1000:1000 {data_dir}") run_cmd(f"chmod -R 755 {data_dir}") # --- Pull Docker image ---------------------------------- if not _validate_docker_image(docker_image): _write_install_status(uuid, "failed", f"Invalid Docker image name: {docker_image}") return _write_install_status(uuid, "pulling", f"Pulling runtime image: {docker_image}") logger.info(f"[{uuid}] Pulling Docker image: {docker_image}") rc, out, err = run_cmd(f"docker pull {docker_image}", timeout=600) if rc != 0: _write_install_status(uuid, "failed", f"Failed to pull image: {err}") return # --- Run install script (if provided) ------------------- if install_script and install_script.strip(): inst_image = install_docker_image or docker_image if inst_image != docker_image: if not _validate_docker_image(inst_image): _write_install_status(uuid, "failed", f"Invalid install image name: {inst_image}") return _write_install_status(uuid, "pulling", f"Pulling install image: {inst_image}") logger.info(f"[{uuid}] Pulling install image: {inst_image}") rc_pull, _, err_pull = run_cmd(f"docker pull {inst_image}", timeout=600) if rc_pull != 0: logger.warning(f"[{uuid}] Failed to pull install image {inst_image}: {err_pull}") _write_install_status(uuid, "failed", f"Failed to pull install image {inst_image}: {err_pull}") return _write_install_status(uuid, "installing", "Running install script...") # Write install script to temp file script_path = server_dir / "install.sh" with open(script_path, "w", newline="\n") as sf: sf.write(install_script) script_path.chmod(0o755) # Build env flags for install inst_env = "" for key, value in environment.items(): safe_key = re.sub(r'[^A-Za-z0-9_]', '', str(key)) safe_val = str(value).replace("'", "'\\''") inst_env += f" -e '{safe_key}={safe_val}'" inst_env += f" -e SERVER_MEMORY={memory_limit}" inst_env += f" -e SERVER_PORT={port or _get_default_port(startup_command, docker_image, environment)}" inst_env += " -e DEBIAN_FRONTEND=noninteractive" install_container = f"{name}-install" run_cmd(f"docker rm -f {install_container} 2>/dev/null") # Detect shell from shebang - then verify it exists in the install image shell = "/bin/sh" first_line = install_script.strip().split("\n")[0] if install_script.strip() else "" if first_line.startswith("#!"): requested_shell = first_line[2:].strip().split()[0] if requested_shell != "/bin/sh": rc_check, _, _ = run_cmd( f"docker run --rm --entrypoint '' {inst_image} test -x {requested_shell} 2>/dev/null", timeout=15, ) if rc_check == 0: shell = requested_shell else: logger.warning( f"Shell {requested_shell} not found in {inst_image}, " f"falling back to /bin/sh and rewriting shebang" ) install_script = install_script.replace( f"#!{requested_shell}", "#!/bin/sh", 1 ) # Fix bash-specific syntax for POSIX sh compatibility # Replace == with = inside [ ] test brackets install_script = re.sub( r'(\[ [^]]*) == ([^]]*\])', r'\1 = \2', install_script, ) with open(script_path, "w", newline="\n") as sf: sf.write(install_script) script_path.chmod(0o755) inst_cmd = ( f"docker run --rm --name {install_container}" f" -v {data_dir}:/mnt/server" f" -v {data_dir}:/home/container" f" -v {script_path}:/mnt/install/install.sh:ro" f" -w /mnt/server" f" {inst_env}" f" {inst_image}" f" {shell} /mnt/install/install.sh" ) logger.info(f"[{uuid}] Running install script with shell {shell}") rc_inst, out_inst, err_inst = run_cmd(inst_cmd, timeout=7200) install_log = (out_inst or "") + ("\n--- STDERR ---\n" + err_inst if err_inst else "") logger.info(f"[{uuid}] Install script output (last 2000 chars): {install_log[-2000:]}") # Fix ownership after install script (runs as root, creates root-owned files) run_cmd(f"chown -R 1000:1000 {data_dir}") if rc_inst != 0: logger.error(f"[{uuid}] Install script FAILED (rc={rc_inst})") run_cmd(f"docker rm -f {install_container} 2>/dev/null", timeout=15) # --- SteamCMD fallback on install failure -------- steam_app_id = None if server_type == ServerTypeDetector.STEAMCMD: steam_app_id = ServerTypeDetector.get_steam_app_id(environment, install_script) if steam_app_id: logger.info(f"[{uuid}] Install failed - SteamCMD fallback for app {steam_app_id}...") _write_install_status(uuid, "installing", f"Install script failed, trying SteamCMD fallback (app {steam_app_id})...") steam_ok, steam_msg = _steamcmd_install(str(data_dir), steam_app_id) if steam_ok: logger.info(f"[{uuid}] SteamCMD fallback succeeded after install script failure") run_cmd(f"chown -R 1000:1000 {data_dir}") else: logger.error(f"[{uuid}] SteamCMD fallback also failed: {steam_msg}") _write_install_status(uuid, "failed", f"Install script failed (rc={rc_inst}) and SteamCMD fallback failed: {steam_msg}", exit_code=rc_inst, install_log=install_log[-3000:]) return else: _write_install_status(uuid, "failed", f"Install script failed with exit code {rc_inst}", exit_code=rc_inst, install_log=install_log[-3000:]) return else: # --- No install script - SteamCMD auto-install if applicable - steam_app_id = None if server_type == ServerTypeDetector.STEAMCMD: steam_app_id = ServerTypeDetector.get_steam_app_id(environment, "") if steam_app_id: logger.info(f"[{uuid}] No install script - SteamCMD auto-install for app {steam_app_id}") _write_install_status(uuid, "installing", f"Auto-installing via SteamCMD (app {steam_app_id})...") steam_ok, steam_msg = _steamcmd_install(str(data_dir), steam_app_id) if not steam_ok: _write_install_status(uuid, "failed", f"SteamCMD auto-install failed: {steam_msg}") return run_cmd(f"chown -R 1000:1000 {data_dir}") # --- Post-install validation ---------------------------- _write_install_status(uuid, "configuring", "Validating installation...") install_issues = _post_install_validate(data_dir, startup_command) if install_issues: for issue in install_issues: logger.warning(f"[{uuid}] Install validation: {issue}") binary = _extract_binary_from_command(startup_command) if binary and (binary.startswith('./') or binary.startswith('/')): bin_rel = binary[2:] if binary.startswith('./') else binary.lstrip('/') bin_path = Path(data_dir) / bin_rel if not bin_path.exists(): # --- SteamCMD fallback for missing binary ---- steam_app_id = None if server_type == ServerTypeDetector.STEAMCMD: steam_app_id = ServerTypeDetector.get_steam_app_id(environment, install_script) if steam_app_id and not any(Path(data_dir).rglob("appmanifest_*.acf")): logger.info(f"[{uuid}] Binary missing - SteamCMD fallback (app {steam_app_id})...") _write_install_status(uuid, "installing", f"Binary missing, SteamCMD fallback for app {steam_app_id}...") steam_ok, steam_msg = _steamcmd_install(str(data_dir), steam_app_id) if steam_ok: run_cmd(f"chown -R 1000:1000 {data_dir}") install_issues = _post_install_validate(data_dir, startup_command) # --- Auto-detect binary ---------------------- bin_path = Path(data_dir) / bin_rel # re-check after fallback if not bin_path.exists(): detected, confidence = _auto_detect_binary(str(data_dir)) if detected: logger.info(f"[{uuid}] Auto-detected binary: {detected} (conf={confidence:.0%})") startup_command = startup_command.replace(binary, detected) # Persist fix to config try: config["startup_command"] = startup_command with open(config_file, "w") as f: json.dump(config, f, indent=2) except Exception: pass else: _write_install_status(uuid, "failed", f"Binary not found after install + SteamCMD fallback + auto-detect: {binary}", issues=install_issues) return # --- Build environment flags ---------------------------- env_flags = "" for key, value in environment.items(): safe_key = re.sub(r'[^A-Za-z0-9_]', '', str(key)) safe_val = str(value).replace("'", "'\\''") env_flags += f" -e '{safe_key}={safe_val}'" mem_bytes = int(memory_limit) * 1024 * 1024 cpu_shares = max(2, int(int(cpu_limit) * 1024 / 100)) # Build the startup command with variable interpolation final_cmd = startup_command for key, value in environment.items(): final_cmd = final_cmd.replace("{{" + str(key) + "}}", str(value)) final_cmd = final_cmd.replace("{{SERVER_MEMORY}}", str(memory_limit)) if port: final_cmd = final_cmd.replace("{{SERVER_PORT}}", str(port)) # Clean up any remaining unresolved {{VARIABLE}} placeholders # These cause errors (e.g. npm tries to install literal "{{ADDITIONAL_PACKAGES}}") unresolved = re.findall(r'\{\{\w+\}\}', final_cmd) if unresolved: logger.info(f"[{uuid}] Cleaning {len(unresolved)} unresolved template vars: {unresolved}") final_cmd = re.sub(r'\{\{\w+\}\}', '', final_cmd) # Clean up empty shell conditionals left behind, e.g.: # if [ ! -z "" ]; then npm install ; fi; # These are harmless but ugly - leave them, shell handles empty strings fine if port: # -- AUTO-INJECT +port FOR SOURCE ENGINE ------------------ cmd_lower = final_cmd.lower() is_source = any(kw in cmd_lower for kw in ["srcds", "gmod", "garrysmod", "csgo", "css", "tf2"]) if is_source and "+port" not in final_cmd: # Add +port parameter if not already present final_cmd = f"{final_cmd} +port {port}" logger.info(f"Auto-injected +port {port} for Source engine server") # -- AUTO-INJECT --port FOR ALL MINECRAFT JAVA SERVERS ------ # CLI --port flag overrides server.properties and is the most reliable # way to ensure the correct port. Works on Vanilla, Paper, Spigot, # Purpur, Forge, Fabric - all MC Java servers accept --port. # Excluded: Bedrock (uses server.properties only), proxies (BungeeCord/Velocity) is_mc_java = _is_minecraft_server(final_cmd, docker_image) is_proxy = any(kw in cmd_lower for kw in ["bungeecord", "velocity", "waterfall", "flamecord", "geyser"]) is_bedrock = "bedrock" in cmd_lower if is_mc_java and not is_proxy and not is_bedrock and "--port" not in final_cmd: final_cmd = f"{final_cmd} --port {port}" logger.info(f"[{uuid}] Auto-injected --port {port} for Minecraft Java server") # Patch known game config files to use the correct port _patch_server_port(data_dir, port or _get_default_port(startup_command, docker_image, environment), startup_command, docker_image) java_runtime_error = _validate_java_runtime_compatibility(data_dir, final_cmd, docker_image) if java_runtime_error: _write_install_status(uuid, "failed", java_runtime_error) logger.error(f"[{uuid}] {java_runtime_error}") return # -- PROACTIVE MINECRAFT EULA FIX ------------------------------ # For Java-based servers, create eula.txt proactively to prevent startup issues img_lower = docker_image.lower() is_java_image = any(kw in img_lower for kw in ["java", "openjdk", "eclipse-temurin", "itzg/"]) if is_java_image: eula_path = Path(data_dir) / "eula.txt" if not eula_path.exists(): try: eula_path.write_text("eula=true\n") logger.info("Proactively created eula.txt for Java-based server") except Exception as e: logger.warning(f"Failed to create proactive eula.txt: {e}") # Fix shell compatibility (#!/bin/bash → #!/bin/sh on images without bash) _fix_shell_compatibility(data_dir, docker_image) # -- AUTO-INJECT npm install FOR Node.js SERVERS ------------ # If startup command uses `node` and package.json exists but # the command doesn't already include npm install, prepend it. node_modules_path = data_dir / "node_modules" package_json_path = data_dir / "package.json" if (package_json_path.exists() and "node " in final_cmd and "npm install" not in final_cmd and "npm ci" not in final_cmd): npm_prefix = ( 'if [ -f /home/container/package.json ] && [ ! -d /home/container/node_modules ]; then ' 'echo "[IKABYTE:FIX] Installing Node.js dependencies..."; ' 'npm install --omit=dev; ' 'fi; ' ) final_cmd = npm_prefix + final_cmd logger.info(f"[{uuid}] Auto-injected npm install for Node.js server") # Build startup wrapper - saved OUTSIDE data dir, mounted read-only container_cmd, wrapper_volume = _build_startup_wrapper(server_dir, data_dir, final_cmd) # Gracefully stop and remove old container if exists # (docker rm -f alone sends SIGKILL - port may stay in TIME_WAIT) run_cmd(f"docker stop -t 10 {name} 2>/dev/null", timeout=20) run_cmd(f"docker rm -f {name} 2>/dev/null") # Remove any leftover .ikabyte_start.sh from data dir (v1.5.x put it there) old_wrapper = data_dir / ".ikabyte_start.sh" if old_wrapper.exists(): try: old_wrapper.unlink() logger.info("Removed old .ikabyte_start.sh from player-visible data dir") except Exception: pass # Ensure data dir ownership for container user run_cmd(f"chown -R 1000:1000 {data_dir}") # --- Create container ----------------------------------- _write_install_status(uuid, "configuring", "Creating container...") docker_cmd = ( f"docker create" f" --name {name}" f" --network host" f" --user 1000:1000" f" --memory={mem_bytes}" f" --cpu-shares={cpu_shares}" f" --pids-limit=512" f" --security-opt=no-new-privileges" f" --restart=no" f" -v {data_dir}:/home/container" f"{wrapper_volume}" f" -w /home/container" f" {env_flags}" f" -e HOME=/home/container" f" -e 'PS1=container@ikabyte~ '" f" -e SERVER_MEMORY={memory_limit}" f" -e SERVER_PORT={port or _get_default_port(startup_command, docker_image, environment)}" f" -e STARTUP={shlex.quote(final_cmd)}" f" --tty" f" --interactive" f" {docker_image}" f" {container_cmd}" ) logger.info(f"[{uuid}] Creating container: {name}") rc, out, err = run_cmd(docker_cmd, timeout=60) if rc != 0: _write_install_status(uuid, "failed", f"Failed to create container: {err}") return # --- Start container ------------------------------------ effective_port = port or _get_default_port(startup_command, docker_image, environment) # Kill TIME_WAIT sockets and wait for port to become free if effective_port: _kill_time_wait_sockets(effective_port) conflict = _check_port_conflict(effective_port) if conflict: logger.warning(f"[{uuid}] Port {effective_port} in use ({conflict.get('state', 'unknown')}) before start - waiting up to 30s...") if not _wait_for_port_free(effective_port, timeout=30): logger.error(f"[{uuid}] Port {effective_port} still occupied: {conflict['process']}") _write_install_status(uuid, "failed", f"Port {effective_port} jest zajęty przez inny proces. " f"Sprawdź czy inny serwer nie używa tego samego portu.") run_cmd(f"docker rm -f {name} 2>/dev/null") return _write_install_status(uuid, "starting", "Starting server...") logger.info(f"[{uuid}] Starting container: {name}") rc_start, out_start, err_start = run_cmd(f"docker start {name}", timeout=30) if rc_start != 0: _write_install_status(uuid, "failed", f"Failed to start container: {err_start}") return # --- Post-start health check --------------------------- health = _post_start_health_check(uuid, name, wait_seconds=8) if not health["healthy"]: # --- Self-healing attempt --------------------------- logger.info(f"[{uuid}] Health check failed - attempting self-healing...") heal = _self_healing_check(uuid, name) if heal.get("actions"): logger.info(f"[{uuid}] Self-healing actions: {heal['actions']}") # Restart with fixes applied run_cmd(f"docker stop -t 5 {name} 2>/dev/null", timeout=15) # Rebuild wrapper with potentially fixed startup command try: with open(config_file) as f: fixed_cfg = json.load(f) fixed_cmd = fixed_cfg.get("startup_command", startup_command) for key, value in environment.items(): fixed_cmd = fixed_cmd.replace("{{" + str(key) + "}}", str(value)) fixed_cmd = fixed_cmd.replace("{{SERVER_MEMORY}}", str(memory_limit)) if port: fixed_cmd = fixed_cmd.replace("{{SERVER_PORT}}", str(port)) _build_startup_wrapper(server_dir, data_dir, fixed_cmd) except Exception: pass run_cmd(f"docker start {name}", timeout=30) health2 = _post_start_health_check(uuid, name, wait_seconds=10) if health2["healthy"]: _write_install_status(uuid, "done", "Server recovered after self-healing", healthy=True, container=name, self_healing=heal["actions"]) logger.info(f"[{uuid}] Self-healing succeeded!") else: _write_install_status(uuid, "done", "Server installed but health check failed even after self-healing", healthy=False, health_check=health2, self_healing=heal["actions"], install_issues=install_issues or None) else: _write_install_status(uuid, "done", "Server installed but health check detected issues", healthy=False, health_check=health, install_issues=install_issues or None) else: _write_install_status(uuid, "done", "Server installed and running successfully", healthy=True, container=name, install_issues=install_issues or None) logger.info(f"[{uuid}] Installation complete - healthy={health.get('healthy', False)}") except Exception as e: logger.error(f"[{uuid}] Install failed with exception: {e}") # Clean up any leftover install container install_container = f"{name}-install" run_cmd(f"docker rm -f {install_container} 2>/dev/null", timeout=15) _write_install_status(uuid, "failed", str(e)) def install_server(data): """ Accept a server install request and run it asynchronously in a background thread. Returns immediately so the HTTP caller doesn't time out. """ uuid = data.get("uuid", "") if not uuid: return {"ok": False, "error": "Missing uuid"} docker_image = data.get("docker_image", "") if not docker_image: return {"ok": False, "error": "Missing docker_image"} # Write initial status so the panel can immediately poll _write_install_status(uuid, "queued", "Install request accepted") # Launch the heavy work in a background thread thread = threading.Thread( target=_install_server_sync, args=(data,), daemon=True, name=f"install-{uuid[:12]}", ) thread.start() logger.info(f"Install queued for {uuid} - running in background thread") return {"ok": True, "status": "installing", "message": "Install started - poll /install-status for progress"} def reinstall_server(uuid, data): """Reinstall a server: stop, wipe data, re-run install, recreate container.""" name = get_container_name(uuid) # Stop and remove existing container run_cmd(f"docker stop -t 5 {name} 2>/dev/null", timeout=15) run_cmd(f"docker rm -f {name} 2>/dev/null", timeout=15) # Wipe server data server_dir = Path(DATA_DIR) / "servers" / uuid data_dir = server_dir / "data" if data_dir.exists(): import shutil shutil.rmtree(data_dir, ignore_errors=True) data_dir.mkdir(parents=True, exist_ok=True) # Re-install using the same flow data["uuid"] = uuid # Load server name from config if available config_file = server_dir / "config.json" if config_file.exists(): with open(config_file) as f: old_config = json.load(f) data.setdefault("name", old_config.get("name", uuid)) return install_server(data) def _rebuild_wrapper_from_config(uuid, server_dir, data_dir, config_file, sync_startup_command=None, sync_environment=None): """ Rebuild the startup wrapper from config.json before starting a server. This ensures any agent-level fixes (npm inject, variable interpolation) are applied even for servers installed with older agent versions. If sync_startup_command is provided (from panel), update config.json with it so the latest egg startup_command is used. If sync_environment is provided, update config.json environment with it so variable changes from panel are reflected. Returns True if wrapper was rebuilt, False on error. """ try: with open(config_file) as f: cfg = json.load(f) except Exception as e: logger.warning(f"[{uuid}] Cannot rebuild wrapper - config unreadable: {e}") return False startup_command = cfg.get("startup_command", "") environment = cfg.get("environment", {}) memory_limit = cfg.get("memory_limit", 1024) port = cfg.get("port") docker_image = cfg.get("docker_image", "") # If the panel sent updated environment variables, merge and persist if sync_environment and isinstance(sync_environment, dict): changed = {k: v for k, v in sync_environment.items() if environment.get(k) != v} if changed: logger.info(f"[{uuid}] Syncing environment from panel: {list(changed.keys())}") environment.update(sync_environment) cfg["environment"] = environment try: with open(config_file, "w") as f: json.dump(cfg, f, indent=2) except Exception: pass # If the panel sent an updated startup_command, use it and persist if sync_startup_command and sync_startup_command != startup_command: logger.info(f"[{uuid}] Syncing startup_command from panel") startup_command = sync_startup_command cfg["startup_command"] = startup_command try: with open(config_file, "w") as f: json.dump(cfg, f, indent=2) except Exception: pass # Variable interpolation (same as install_server) final_cmd = startup_command for key, value in environment.items(): final_cmd = final_cmd.replace("{{" + str(key) + "}}", str(value)) final_cmd = final_cmd.replace("{{SERVER_MEMORY}}", str(memory_limit)) if port: final_cmd = final_cmd.replace("{{SERVER_PORT}}", str(port)) # Clean up any remaining unresolved {{VARIABLE}} placeholders unresolved = re.findall(r'\{\{\w+\}\}', final_cmd) if unresolved: logger.info(f"[{uuid}] Cleaning {len(unresolved)} unresolved template vars: {unresolved}") final_cmd = re.sub(r'\{\{\w+\}\}', '', final_cmd) if port: # Auto-inject +port for Source engine servers cmd_lower = final_cmd.lower() is_source = any(kw in cmd_lower for kw in ["srcds", "gmod", "garrysmod", "csgo", "css", "tf2"]) if is_source and "+port" not in final_cmd: final_cmd = f"{final_cmd} +port {port}" logger.info(f"[{uuid}] Auto-injected +port {port} for Source engine server (wrapper rebuild)") # Auto-inject --port for Bukkit-based servers (Paper/Spigot/Purpur) # NOTE: Forge, Fabric, BungeeCord, Velocity do NOT support --port if _is_bukkit_based_server(final_cmd, docker_image, data_dir) and "--port" not in final_cmd: final_cmd = f"{final_cmd} --port {port}" logger.info(f"[{uuid}] Auto-injected --port {port} for Bukkit-based server (wrapper rebuild)") # Auto-inject npm install for Node.js servers package_json_path = Path(data_dir) / "package.json" if (package_json_path.exists() and "node " in final_cmd and "npm install" not in final_cmd and "npm ci" not in final_cmd): npm_prefix = ( 'if [ -f /home/container/package.json ] && [ ! -d /home/container/node_modules ]; then ' 'echo "[IKABYTE:FIX] Installing Node.js dependencies..."; ' 'npm install --omit=dev; ' 'fi; ' ) final_cmd = npm_prefix + final_cmd logger.info(f"[{uuid}] Auto-injected npm install for Node.js server (wrapper rebuild)") # Rebuild the wrapper file (it's mounted as a volume, Docker sees the update) _build_startup_wrapper(server_dir, data_dir, final_cmd) logger.info(f"[{uuid}] Rebuilt startup wrapper before start") return True def power_action(uuid, action, sync_config=None): """Perform a power action on a game server.""" name = get_container_name(uuid) if action == "start": status = docker_server_status(uuid) if status == "not_found": return {"ok": False, "error": "Server not installed"} # Block start if server was fraud-suspended if uuid in _fraud_suspended_servers: return {"ok": False, "error": "Serwer zawieszony za oszustwo - skontaktuj się z pomocą techniczną"} # Patch port in config files before starting server_dir = Path(DATA_DIR) / "servers" / uuid config_file = server_dir / "config.json" data_dir = server_dir / "data" docker_image = None startup_cmd_cfg = None p = None if config_file.exists(): try: with open(config_file) as f: cfg = json.load(f) p = cfg.get("port") if p: _patch_server_port(data_dir, p or _get_default_port(cfg.get("startup_command", ""), cfg.get("docker_image", ""), cfg.get("environment", {})), cfg.get("startup_command", ""), cfg.get("docker_image", "")) docker_image = cfg.get("docker_image") except Exception: pass # Rebuild startup wrapper with latest agent fixes (npm inject, etc.) sync_cmd = sync_config.get("startup_command") if sync_config else None sync_env = sync_config.get("environment") if sync_config else None _rebuild_wrapper_from_config(uuid, server_dir, data_dir, config_file, sync_startup_command=sync_cmd, sync_environment=sync_env) # Fix shell compatibility before starting (handles #!/bin/bash on Alpine) _fix_shell_compatibility(data_dir, docker_image) # Check for port conflicts before starting if p: _kill_time_wait_sockets(p) conflict = _check_port_conflict(p) if conflict: logger.warning(f"[{uuid}] Port {p} in use ({conflict.get('state', 'unknown')}) - waiting up to 30s...") if not _wait_for_port_free(p, timeout=30): return {"ok": False, "error": f"Port {p} jest zajęty przez inny proces"} rc, out, err = run_cmd(f"docker start {name}", timeout=30) if rc == 0: # Clear "stopped" status so self-healing knows this is intentionally running _write_install_status(uuid, "done", "Server started by user", healthy=True) # Queue fraud scan 30s after start _queue_fraud_scan(uuid, delay=30) # LinuxGSM: re-apply port config and restart message injector if rc == 0 and _is_linuxgsm_container(uuid): try: lgsm_game = cfg.get("environment", {}).get("LGSM_GAMESERVER", "") p = cfg.get("port") if lgsm_game and p: safe_lgsm = re.sub(r'[^a-zA-Z0-9_-]', '', lgsm_game) _write_linuxgsm_port_config(uuid, name, safe_lgsm, p, server_dir) except Exception: pass _start_linuxgsm_message_injector(uuid, name) return {"ok": rc == 0, "message": out if rc == 0 else err} elif action == "stop": # Stop LinuxGSM message injector if running if uuid in _lgsm_injectors: _lgsm_injectors[uuid].set() # Read port before stopping to kill TIME_WAIT sockets after stop_port = None try: stop_cfg_file = Path(DATA_DIR) / "servers" / uuid / "config.json" if stop_cfg_file.exists(): with open(stop_cfg_file) as f: stop_port = json.load(f).get("port") except Exception: pass rc, out, err = run_cmd(f"docker stop -t 10 {name}", timeout=25) if rc != 0: # Graceful stop failed (SteamCMD servers often ignore SIGTERM) - force kill logging.warning(f"Graceful stop failed for {name}, forcing kill: {err}") rc2, out2, err2 = run_cmd(f"docker kill {name}", timeout=10) if rc2 == 0: rc, out, err = rc2, out2, err2 if rc == 0: # Kill TIME_WAIT sockets immediately so next start doesn't have to wait if stop_port: _kill_time_wait_sockets(stop_port) # Mark as intentionally stopped so self-healing won't restart it _write_install_status(uuid, "stopped", "Server stopped by user") return {"ok": rc == 0, "message": out if rc == 0 else err} elif action == "restart": # Patch port & fix shell compatibility before restarting server_dir = Path(DATA_DIR) / "servers" / uuid config_file = server_dir / "config.json" data_dir = server_dir / "data" docker_image = None effective_port = None if config_file.exists(): try: with open(config_file) as f: cfg = json.load(f) effective_port = cfg.get("port") if effective_port: _patch_server_port(data_dir, effective_port or _get_default_port(cfg.get("startup_command", ""), cfg.get("docker_image", ""), cfg.get("environment", {})), cfg.get("startup_command", ""), cfg.get("docker_image", "")) docker_image = cfg.get("docker_image") except Exception: pass # Rebuild wrapper with latest fixes sync_cmd = sync_config.get("startup_command") if sync_config else None sync_env = sync_config.get("environment") if sync_config else None _rebuild_wrapper_from_config(uuid, server_dir, data_dir, config_file, sync_startup_command=sync_cmd, sync_environment=sync_env) _fix_shell_compatibility(data_dir, docker_image) # Explicit stop → kill TIME_WAIT → start (instead of docker restart) # docker restart can race: new process binds before old one releases the port rc_stop, _, err_stop = run_cmd(f"docker stop -t 10 {name}", timeout=25) if rc_stop != 0: logging.warning(f"Graceful stop failed for {name} during restart, forcing kill: {err_stop}") run_cmd(f"docker kill {name}", timeout=10) if effective_port: _kill_time_wait_sockets(effective_port) if not _wait_for_port_free(effective_port, timeout=30): logging.warning(f"Port {effective_port} still occupied after 30s for {name}, attempting start anyway") rc, out, err = run_cmd(f"docker start {name}", timeout=30) # Restart LinuxGSM message injector if applicable if rc == 0 and _is_linuxgsm_container(uuid): _start_linuxgsm_message_injector(uuid, name) return {"ok": rc == 0, "message": out if rc == 0 else err} elif action == "kill": rc, out, err = run_cmd(f"docker kill {name}", timeout=10) if rc == 0: _write_install_status(uuid, "stopped", "Server killed by user") return {"ok": rc == 0, "message": out if rc == 0 else err} return {"ok": False, "error": f"Unknown action: {action}"} def send_command(uuid, command): """Send a console command to a running game server.""" name = get_container_name(uuid) status = docker_server_status(uuid) if status != "running": return {"ok": False, "error": "Server is not running"} # Send command via docker exec piping to the STDIN of the main process safe_cmd = shlex.quote(command) rc, out, err = run_cmd( f"docker exec {name} /bin/sh -c 'echo {safe_cmd} > /proc/1/fd/0'", timeout=10, ) # Fallback: use docker exec with a different approach if rc != 0: rc, out, err = run_cmd( f"docker exec -i {name} sh -c 'echo {safe_cmd}'", timeout=10, ) return {"ok": True, "message": "Command sent"} def get_console_logs(uuid): """Get console output from a game server.""" name = get_container_name(uuid) rc, out, err = run_cmd( f"docker logs --tail {CONSOLE_LOG_LINES} --timestamps {name} 2>&1", timeout=10, ) if rc == 0: lines = out.splitlines() if out else [] return {"ok": True, "logs": lines[-CONSOLE_LOG_LINES:]} return {"ok": True, "logs": []} def stream_console_logs(handler, uuid): """Stream console logs as SSE (Server-Sent Events) via docker logs -f. IMPORTANT: Max 5-minute connection timeout to prevent resource exhaustion. Clients should reconnect after timeout - this prevents thread pool starvation. """ name = get_container_name(uuid) # Check if container exists rc, _, _ = run_cmd(f"docker inspect {name} 2>/dev/null", timeout=5) if rc != 0: handler.send_response(200) handler.send_header("Content-Type", "text/event-stream") handler.send_header("Cache-Control", "no-cache") handler.send_header("Connection", "keep-alive") handler.send_header("X-Agent-Version", AGENT_VERSION) handler.end_headers() handler.wfile.write(b"data: {\"error\": \"container_not_found\"}\n\n") handler.wfile.flush() return handler.send_response(200) handler.send_header("Content-Type", "text/event-stream") handler.send_header("Cache-Control", "no-cache") handler.send_header("Connection", "keep-alive") handler.send_header("X-Agent-Version", AGENT_VERSION) handler.end_headers() # First, send existing logs as a batch rc, out, _ = run_cmd( f"docker logs --tail {CONSOLE_LOG_LINES} --timestamps {name} 2>&1", timeout=10, ) if rc == 0 and out: lines = out.splitlines()[-CONSOLE_LOG_LINES:] batch = json.dumps({"batch": lines}) handler.wfile.write(f"data: {batch}\n\n".encode("utf-8")) handler.wfile.flush() # Then stream new lines using docker logs -f --since now proc = subprocess.Popen( ["docker", "logs", "-f", "--since", "0s", "--timestamps", name], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) heartbeat_interval = 15 last_heartbeat = time.time() stream_start = time.time() max_stream_duration = 300 # 5 minutes max per connection try: import select while True: # Check if stream has exceeded max duration elapsed = time.time() - stream_start if elapsed > max_stream_duration: logger.debug(f"[{uuid}] SSE stream timeout ({max_stream_duration}s) - closing connection") handler.wfile.write(b"data: {\"error\": \"stream_ended\", \"reason\": \"timeout\"}\n\n") handler.wfile.flush() break # Use select with timeout for heartbeats ready, _, _ = select.select([proc.stdout], [], [], 1.0) if ready: raw = proc.stdout.readline() if not raw: break line = raw.decode("utf-8", errors="replace").rstrip("\n\r") if line: data = json.dumps({"line": line}) handler.wfile.write(f"data: {data}\n\n".encode("utf-8")) handler.wfile.flush() # Send heartbeat to keep connection alive now = time.time() if now - last_heartbeat >= heartbeat_interval: handler.wfile.write(b": heartbeat\n\n") handler.wfile.flush() last_heartbeat = now # Check if process is still alive if proc.poll() is not None: break except (BrokenPipeError, ConnectionResetError, OSError): pass finally: proc.kill() proc.wait() def _get_disk_bytes(uuid): """Get actual disk usage for a server's data directory (cached).""" import time as _time now = _time.time() cached = _disk_bytes_cache.get(uuid) if cached and now - cached["ts"] < DISK_CACHE_TTL: return cached["bytes"] data_dir = os.path.join(DATA_DIR, "servers", uuid, "data") if not os.path.isdir(data_dir): return 0 rc, out, _ = run_cmd(f"du -sb {shlex.quote(data_dir)} 2>/dev/null", timeout=10) if rc == 0 and out.strip(): try: val = int(out.split()[0]) _disk_bytes_cache[uuid] = {"bytes": val, "ts": now} return val except (ValueError, IndexError): pass return cached["bytes"] if cached else 0 def _read_proc_net_dev(pid): """Read /proc//net/dev and sum rx/tx bytes for non-lo interfaces.""" try: with open(f"/proc/{pid}/net/dev") as f: lines = f.readlines() rx_total = 0 tx_total = 0 for line in lines[2:]: # Skip 2 header lines parts = line.split() if len(parts) < 10: continue iface = parts[0].rstrip(":") if iface == "lo": continue rx_total += int(parts[1]) # rx_bytes column tx_total += int(parts[9]) # tx_bytes column return rx_total, tx_total except (FileNotFoundError, PermissionError, ValueError, IndexError): return 0, 0 def _get_host_net_stats(uuid, container_name, started_at): """Get network stats for host-networked containers via /proc/net/dev baseline tracking. Docker reports 0 for network I/O in --network=host mode. This function reads /proc//net/dev and tracks delta since container start. Note: on multi-container nodes this includes all host traffic, not per-container. """ rc, pid_out, _ = run_cmd( f"docker inspect --format '{{{{.State.Pid}}}}' {container_name} 2>/dev/null", timeout=3 ) if rc != 0 or not pid_out.strip(): return 0, 0 pid = pid_out.strip().strip("'") if not pid or pid == "0": return 0, 0 current_rx, current_tx = _read_proc_net_dev(pid) if current_rx == 0 and current_tx == 0: return 0, 0 baseline = _net_baselines.get(uuid) if baseline is None or baseline.get("started_at") != started_at: # First call for this container instance - save baseline _net_baselines[uuid] = {"rx": current_rx, "tx": current_tx, "started_at": started_at} return 0, 0 return max(0, current_rx - baseline["rx"]), max(0, current_tx - baseline["tx"]) def get_container_stats(uuid): """Get real-time Docker container resource usage stats.""" name = get_container_name(uuid) # Check if installation is still in progress - if so, report 'installing' # regardless of the game container's Docker state. install_status = _read_install_status(uuid) if install_status.get("status") in ("pulling", "installing", "configuring", "starting"): return { "ok": True, "state": "installing", "cpu_percent": 0, "memory_bytes": 0, "memory_limit_bytes": 0, "memory_percent": 0, "network_rx_bytes": 0, "network_tx_bytes": 0, "disk_read_bytes": 0, "disk_write_bytes": 0, "pids": 0, "uptime_seconds": 0, } # If this server was fraud-suspended by the scanner, report it in stats # so the panel can sync suspension state even if _report_fraud_to_panel failed if uuid in _fraud_suspended_servers: report_path = Path(DATA_DIR) / "servers" / uuid / "fraud_report.json" fraud_reason = "Wykryto próbę oszustwa wymagany kontakt z centrum pomocy" fraud_at = None try: if report_path.exists(): with open(report_path) as f: report = json.load(f) fraud_reason = report.get("suspend_reason", fraud_reason) fraud_at = report.get("suspended_at") except Exception: pass return { "ok": True, "state": "suspended", "is_suspended": True, "fraud_suspended": True, "suspend_reason": fraud_reason, "suspended_at": fraud_at, "cpu_percent": 0, "memory_bytes": 0, "memory_limit_bytes": 0, "memory_percent": 0, "network_rx_bytes": 0, "network_tx_bytes": 0, "disk_read_bytes": 0, "disk_write_bytes": 0, "pids": 0, "uptime_seconds": 0, } # Check if container is running rc_state, state_out, _ = run_cmd( f"docker inspect --format '{{{{.State.Status}}}}' {name} 2>/dev/null", timeout=5 ) container_state = state_out.strip().strip("'") if rc_state == 0 else "unknown" if container_state != "running": result = { "ok": True, "state": container_state, "cpu_percent": 0, "memory_bytes": 0, "memory_limit_bytes": 0, "memory_percent": 0, "network_rx_bytes": 0, "network_tx_bytes": 0, "disk_read_bytes": 0, "disk_write_bytes": 0, "pids": 0, "uptime_seconds": 0, } # For crashed/exited containers, include last log lines for debugging if container_state in ("exited", "dead"): rc_log, logs, _ = run_cmd(f"docker logs --tail 50 {name} 2>&1", timeout=5) if rc_log == 0 and logs: result["crash_log"] = logs.strip().split("\n")[-50:] return result # Get stats (no-stream = single snapshot) rc, out, err = run_cmd( f"docker stats --no-stream --format '{{{{json .}}}}' {name}", timeout=5, ) stats = { "ok": True, "state": container_state, "cpu_percent": 0, "memory_bytes": 0, "memory_limit_bytes": 0, "memory_percent": 0, "network_rx_bytes": 0, "network_tx_bytes": 0, "disk_read_bytes": 0, "disk_write_bytes": 0, "pids": 0, "uptime_seconds": 0, } if rc == 0 and out: try: raw = json.loads(out.strip().strip("'")) # Parse CPU cpu_str = raw.get("CPUPerc", "0%").replace("%", "") stats["cpu_percent"] = round(float(cpu_str), 2) # Parse Memory usage / limit mem_usage = raw.get("MemUsage", "0B / 0B") parts = mem_usage.split("/") if len(parts) == 2: stats["memory_bytes"] = _parse_docker_size(parts[0].strip()) stats["memory_limit_bytes"] = _parse_docker_size(parts[1].strip()) mem_pct = raw.get("MemPerc", "0%").replace("%", "") stats["memory_percent"] = round(float(mem_pct), 2) # Parse Network I/O net_io = raw.get("NetIO", "0B / 0B") net_parts = net_io.split("/") if len(net_parts) == 2: stats["network_rx_bytes"] = _parse_docker_size(net_parts[0].strip()) stats["network_tx_bytes"] = _parse_docker_size(net_parts[1].strip()) # Parse Block I/O block_io = raw.get("BlockIO", "0B / 0B") block_parts = block_io.split("/") if len(block_parts) == 2: stats["disk_read_bytes"] = _parse_docker_size(block_parts[0].strip()) stats["disk_write_bytes"] = _parse_docker_size(block_parts[1].strip()) # PIDs stats["pids"] = int(raw.get("PIDs", 0)) except (json.JSONDecodeError, ValueError, KeyError) as e: logger.warning(f"Failed to parse docker stats: {e}") # Get uptime rc_up, up_out, _ = run_cmd( f"docker inspect --format '{{{{.State.StartedAt}}}}' {name} 2>/dev/null", timeout=5 ) if rc_up == 0 and up_out.strip(): try: from datetime import datetime, timezone started = up_out.strip().strip("'") # Docker returns ISO 8601 with nanoseconds if "." in started: started = started[:started.index(".")] + "+00:00" start_dt = datetime.fromisoformat(started) now = datetime.now(timezone.utc) stats["uptime_seconds"] = int((now - start_dt).total_seconds()) except Exception: pass # Compute actual disk usage (du -sb, cached) stats["disk_bytes"] = _get_disk_bytes(uuid) # Fix network stats for --network=host containers (Docker returns 0) if stats["network_rx_bytes"] == 0 and stats["network_tx_bytes"] == 0: started_at = up_out.strip().strip("'") if rc_up == 0 else "" fallback_rx, fallback_tx = _get_host_net_stats(uuid, name, started_at) stats["network_rx_bytes"] = fallback_rx stats["network_tx_bytes"] = fallback_tx return stats def _parse_docker_size(s): """Parse Docker size strings like '1.5GiB', '256MiB', '1.2kB' into bytes.""" s = s.strip() multipliers = { 'B': 1, 'kB': 1000, 'KB': 1000, 'MB': 1000000, 'MiB': 1048576, 'GB': 1000000000, 'GiB': 1073741824, 'TB': 1000000000000, 'TiB': 1099511627776, } for suffix, mult in sorted(multipliers.items(), key=lambda x: -len(x[0])): if s.endswith(suffix): try: return int(float(s[:-len(suffix)].strip()) * mult) except ValueError: return 0 try: return int(float(s)) except ValueError: return 0 def delete_server(uuid): """Delete a game server container, SFTP user, and data.""" name = get_container_name(uuid) # Stop container if running run_cmd(f"docker stop -t 5 {name} 2>/dev/null", timeout=15) # Remove container run_cmd(f"docker rm -f {name} 2>/dev/null", timeout=15) # Delete SFTP user for this server delete_sftp_user(uuid) # Remove SFTP chroot symlink short = uuid[:12].replace('-', '') sftp_link = Path(DATA_DIR) / "servers" / f"sftp_{short}" if sftp_link.is_symlink(): run_cmd(f"rm -f {sftp_link}") # Remove server data directory server_dir = Path(DATA_DIR) / "servers" / uuid if server_dir.exists(): import shutil shutil.rmtree(server_dir, ignore_errors=True) logger.info(f"Server data removed: {server_dir}") return {"ok": True, "message": "Server and data removed"} # --- File Management --------------------------------------------- def _get_server_data_dir(uuid): """Get the server data directory, ensuring path safety.""" safe_uuid = re.sub(r'[^a-zA-Z0-9_-]', '', uuid) return Path(DATA_DIR) / "servers" / safe_uuid / "data" def _safe_path(base_dir, relative_path): """Resolve a relative path safely under base_dir, preventing path traversal.""" base = Path(base_dir).resolve() target = (base / relative_path.lstrip("/")).resolve() if not str(target).startswith(str(base)): return None return target def list_files(uuid, path="/"): """List files and directories in a server's data directory.""" data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} logger.info(f"list_files uuid={uuid} path={path} target={target}") if not target.exists(): return {"ok": True, "files": []} if not target.is_dir(): return {"ok": False, "error": "Not a directory"} files = [] try: for entry in sorted(target.iterdir(), key=lambda e: (not e.is_dir(), e.name.lower())): stat = entry.stat() files.append({ "name": entry.name, "is_directory": entry.is_dir(), "size": stat.st_size if entry.is_file() else None, "modified": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(stat.st_mtime)), }) except PermissionError: return {"ok": False, "error": "Permission denied"} return {"ok": True, "files": files} def read_file_content(uuid, path): """Read the content of a file in a server's data directory.""" data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} if not target.exists(): return {"ok": False, "error": "File not found"} if not target.is_file(): return {"ok": False, "error": "Not a file"} # Max 5MB for text editing if target.stat().st_size > 5 * 1024 * 1024: return {"ok": False, "error": "File too large (max 5MB)"} try: content = target.read_text(encoding="utf-8", errors="replace") return {"ok": True, "content": content} except Exception as e: return {"ok": False, "error": str(e)} def write_file_content(uuid, path, content, encoding=None): """Write content to a file in a server's data directory.""" if not path or path.strip('/') == '': return {"ok": False, "error": "Path required"} data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} logger.info(f"write_file uuid={uuid} path={path} target={target} encoding={encoding}") try: import base64 target.parent.mkdir(parents=True, exist_ok=True) if encoding == 'base64': raw = base64.b64decode(content) target.write_bytes(raw) else: target.write_text(content, encoding="utf-8") return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def create_directory(uuid, path): """Create a directory in a server's data directory.""" if not path or path.strip('/') == '': return {"ok": False, "error": "Path required"} data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} if target.resolve() == Path(data_dir).resolve(): return {"ok": False, "error": "Cannot create root directory"} logger.info(f"create_directory uuid={uuid} path={path} target={target}") try: target.mkdir(parents=True, exist_ok=True) return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def delete_path(uuid, path): """Delete a file or directory in a server's data directory.""" data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} if not target.exists(): return {"ok": False, "error": "Path not found"} # Prevent deleting the root data directory if target.resolve() == data_dir.resolve(): return {"ok": False, "error": "Cannot delete root directory"} try: if target.is_dir(): import shutil shutil.rmtree(target) else: target.unlink() return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def rename_path(uuid, path, new_name): """Rename a file or directory in a server's data directory.""" if not path or path.strip('/') == '': return {"ok": False, "error": "Path required"} if not new_name or '/' in new_name or new_name in ('.', '..'): return {"ok": False, "error": "Invalid new name"} data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} if not target.exists(): return {"ok": False, "error": "Path not found"} if target.resolve() == Path(data_dir).resolve(): return {"ok": False, "error": "Cannot rename root directory"} new_target = target.parent / new_name if new_target.exists(): return {"ok": False, "error": "A file or folder with that name already exists"} # Validate new path is still within data dir try: new_target.resolve().relative_to(Path(data_dir).resolve()) except ValueError: return {"ok": False, "error": "Invalid new name"} logger.info(f"rename_path uuid={uuid} path={path} new_name={new_name}") try: target.rename(new_target) return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def move_path(uuid, source, destination): """Move a file or directory to a new location in a server's data directory.""" if not source or source.strip('/') == '': return {"ok": False, "error": "Source path required"} if not destination and destination != '/': return {"ok": False, "error": "Destination path required"} data_dir = _get_server_data_dir(uuid) src = _safe_path(data_dir, source) if src is None: return {"ok": False, "error": "Invalid source path"} if not src.exists(): return {"ok": False, "error": "Source not found"} if src.resolve() == Path(data_dir).resolve(): return {"ok": False, "error": "Cannot move root directory"} dst_dir = _safe_path(data_dir, destination) if dst_dir is None: return {"ok": False, "error": "Invalid destination path"} if not dst_dir.exists(): return {"ok": False, "error": "Destination directory not found"} if not dst_dir.is_dir(): return {"ok": False, "error": "Destination is not a directory"} final = dst_dir / src.name if final.exists(): return {"ok": False, "error": "A file or folder with that name already exists in the destination"} logger.info(f"move_path uuid={uuid} source={source} destination={destination}") try: import shutil shutil.move(str(src), str(final)) return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def bulk_delete(uuid, root, files): """Delete multiple files/directories in a server's data directory.""" data_dir = _get_server_data_dir(uuid) root_dir = _safe_path(data_dir, root) if root_dir is None: return {"ok": False, "error": "Invalid root path"} if not root_dir.exists() or not root_dir.is_dir(): return {"ok": False, "error": "Root directory not found"} errors = [] deleted = 0 for fname in files: fpath = _safe_path(data_dir, root.rstrip("/") + "/" + fname) if fpath is None: errors.append(f"{fname}: invalid path") continue if not fpath.exists(): errors.append(f"{fname}: not found") continue if fpath.resolve() == Path(data_dir).resolve(): errors.append(f"{fname}: cannot delete root") continue try: if fpath.is_dir(): import shutil shutil.rmtree(fpath) else: fpath.unlink() deleted += 1 except Exception as e: errors.append(f"{fname}: {str(e)}") if errors: return {"ok": True, "deleted": deleted, "errors": errors} return {"ok": True, "deleted": deleted} def bulk_move(uuid, root, files, destination): """Move multiple files/directories to a new location.""" data_dir = _get_server_data_dir(uuid) root_dir = _safe_path(data_dir, root) if root_dir is None: return {"ok": False, "error": "Invalid root path"} dst_dir = _safe_path(data_dir, destination) if dst_dir is None: return {"ok": False, "error": "Invalid destination path"} if not dst_dir.exists() or not dst_dir.is_dir(): return {"ok": False, "error": "Destination directory not found"} errors = [] moved = 0 for fname in files: fpath = _safe_path(data_dir, root.rstrip("/") + "/" + fname) if fpath is None: errors.append(f"{fname}: invalid path") continue if not fpath.exists(): errors.append(f"{fname}: not found") continue final = dst_dir / fpath.name if final.exists(): errors.append(f"{fname}: already exists in destination") continue try: import shutil shutil.move(str(fpath), str(final)) moved += 1 except Exception as e: errors.append(f"{fname}: {str(e)}") if errors: return {"ok": True, "moved": moved, "errors": errors} return {"ok": True, "moved": moved} def compress_files(uuid, root, files): """Compress files into a zip archive in a server's data directory.""" import tarfile as _tarfile data_dir = _get_server_data_dir(uuid) root_dir = _safe_path(data_dir, root) if root_dir is None: return {"ok": False, "error": "Invalid root path"} if not root_dir.exists() or not root_dir.is_dir(): return {"ok": False, "error": "Root directory not found"} # Generate archive name import datetime ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") archive_name = f"archive_{ts}.tar.gz" archive_path = root_dir / archive_name logger.info(f"compress_files uuid={uuid} root={root} files={files} -> {archive_path}") try: with _tarfile.open(str(archive_path), "w:gz") as tar: for fname in files: fpath = _safe_path(data_dir, root.rstrip("/") + "/" + fname) if fpath is None: continue if not fpath.exists(): continue tar.add(str(fpath), arcname=fname) return {"ok": True, "file": archive_name} except Exception as e: return {"ok": False, "error": str(e)} def decompress_file(uuid, root, file): """Decompress an archive in a server's data directory.""" import tarfile as _tarfile data_dir = _get_server_data_dir(uuid) root_dir = _safe_path(data_dir, root) if root_dir is None: return {"ok": False, "error": "Invalid root path"} file_path = _safe_path(data_dir, root.rstrip("/") + "/" + file) if file_path is None: return {"ok": False, "error": "Invalid file path"} if not file_path.exists(): return {"ok": False, "error": "File not found"} logger.info(f"decompress_file uuid={uuid} root={root} file={file}") try: name_lower = file.lower() if name_lower.endswith('.zip'): with zipfile.ZipFile(str(file_path), 'r') as zf: # Path traversal protection for info in zf.infolist(): target = _safe_path(str(root_dir), info.filename) if target is None: return {"ok": False, "error": f"Unsafe path in archive: {info.filename}"} zf.extractall(str(root_dir)) elif name_lower.endswith(('.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.tar')): mode = 'r:gz' if name_lower.endswith(('.tar.gz', '.tgz')) else \ 'r:bz2' if name_lower.endswith('.tar.bz2') else \ 'r:xz' if name_lower.endswith('.tar.xz') else 'r:' with _tarfile.open(str(file_path), mode) as tf: # Path traversal protection for member in tf.getmembers(): target = _safe_path(str(root_dir), member.name) if target is None: return {"ok": False, "error": f"Unsafe path in archive: {member.name}"} tf.extractall(str(root_dir), filter='data') elif name_lower.endswith('.gz'): import gzip out_name = file[:-3] if file.lower().endswith('.gz') else file + '.out' out_path = root_dir / out_name with gzip.open(str(file_path), 'rb') as gz_in: out_path.write_bytes(gz_in.read()) else: return {"ok": False, "error": "Unsupported archive format"} return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def get_file_download_url(uuid, path): """Generate a one-time download token for a file.""" data_dir = _get_server_data_dir(uuid) target = _safe_path(data_dir, path) if target is None: return {"ok": False, "error": "Invalid path"} if not target.exists() or not target.is_file(): return {"ok": False, "error": "File not found"} # Generate a token and store download mapping import secrets token = secrets.token_urlsafe(32) _download_tokens[token] = {"path": str(target), "uuid": uuid, "created": time.time()} url = f"/api/download/{token}" return {"ok": True, "url": url} # --- Backup Management -------------------------------------------- def _get_backup_dir(uuid): """Get the backup directory for a server, creating if needed.""" safe_uuid = re.sub(r'[^a-zA-Z0-9_-]', '', uuid) backup_dir = Path(DATA_DIR) / "servers" / safe_uuid / "backups" backup_dir.mkdir(parents=True, exist_ok=True) return backup_dir def create_backup(uuid, backup_uuid, backup_name=""): """Create a tar.gz backup of the server's data directory.""" import tarfile data_dir = _get_server_data_dir(uuid) if not data_dir.exists(): return {"ok": False, "error": "Server data directory not found"} safe_backup_uuid = re.sub(r'[^a-zA-Z0-9_-]', '', backup_uuid) backup_dir = _get_backup_dir(uuid) backup_path = backup_dir / f"{safe_backup_uuid}.tar.gz" try: with tarfile.open(str(backup_path), "w:gz") as tar: tar.add(str(data_dir), arcname=".") size = backup_path.stat().st_size return {"ok": True, "size": size, "path": str(backup_path)} except Exception as e: if backup_path.exists(): backup_path.unlink() return {"ok": False, "error": str(e)} def list_backups(uuid): """List all backups for a server.""" backup_dir = _get_backup_dir(uuid) backups = [] try: for entry in sorted(backup_dir.iterdir(), key=lambda e: e.stat().st_mtime, reverse=True): if entry.is_file() and entry.name.endswith(".tar.gz"): stat = entry.stat() backups.append({ "uuid": entry.stem.replace(".tar", ""), "size": stat.st_size, "created_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(stat.st_mtime)), }) except Exception: pass return {"ok": True, "backups": backups} def delete_backup(uuid, backup_uuid): """Delete a specific backup.""" safe_backup_uuid = re.sub(r'[^a-zA-Z0-9_-]', '', backup_uuid) backup_dir = _get_backup_dir(uuid) backup_path = backup_dir / f"{safe_backup_uuid}.tar.gz" if not backup_path.exists(): return {"ok": False, "error": "Backup not found"} try: backup_path.unlink() return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def restore_backup(uuid, backup_uuid): """Restore a backup to the server's data directory.""" import tarfile import shutil safe_backup_uuid = re.sub(r'[^a-zA-Z0-9_-]', '', backup_uuid) backup_dir = _get_backup_dir(uuid) backup_path = backup_dir / f"{safe_backup_uuid}.tar.gz" data_dir = _get_server_data_dir(uuid) if not backup_path.exists(): return {"ok": False, "error": "Backup not found"} # Check server is not running status = docker_server_status(uuid) if status == "running": return {"ok": False, "error": "Server must be stopped before restoring"} try: # Clear data dir if data_dir.exists(): shutil.rmtree(data_dir) data_dir.mkdir(parents=True, exist_ok=True) with tarfile.open(str(backup_path), "r:gz") as tar: # Security: prevent path traversal in tar for member in tar.getmembers(): member_path = (data_dir / member.name).resolve() if not str(member_path).startswith(str(data_dir.resolve())): return {"ok": False, "error": "Malicious backup detected (path traversal)"} tar.extractall(str(data_dir)) return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} # --- SFTP User Management --------------------------------- def ensure_sftp_ready(): """Ensure SSH server is installed and configured for SFTP-only access.""" # Install openssh-server and acl if not present rc, _, _ = run_cmd("which sshd") if rc != 0: run_cmd("apt-get update -qq && apt-get install -y -qq openssh-server acl", timeout=120) run_cmd("mkdir -p /run/sshd") else: # Ensure acl is installed even if sshd already exists rc2, _, _ = run_cmd("which setfacl") if rc2 != 0: run_cmd("apt-get update -qq && apt-get install -y -qq acl", timeout=60) sshd_config = "/etc/ssh/sshd_config" marker = "# IkaByte SFTP Config v3" # Check if already configured (with current version) rc, out, _ = run_cmd(f"grep -c '{marker}' {sshd_config}") if rc == 0 and out.strip() != "0": return # Already configured with v3 # Remove old config block if present run_cmd(f"sed -i '/# IkaByte SFTP Config/,/^$/d' {sshd_config}") run_cmd(f"sed -i '/^Match User sftp_/,/^$/d' {sshd_config}") # Disable banners that break SFTP protocol run_cmd(f"sed -i 's/^PrintMotd .*/PrintMotd no/' {sshd_config}") run_cmd(f"sed -i 's/^Banner .*/Banner none/' {sshd_config}") # Ensure internal-sftp subsystem run_cmd(f"sed -i 's|^Subsystem.*sftp.*|Subsystem sftp internal-sftp|' {sshd_config}") rc2, out2, _ = run_cmd(f"grep -c 'Subsystem.*sftp' {sshd_config}") if rc2 != 0 or out2.strip() == "0": run_cmd(f"echo 'Subsystem sftp internal-sftp' >> {sshd_config}") # Add SFTP match block for sftp_ users with chroot servers_dir = Path(DATA_DIR) / "servers" sftp_block = f""" {marker} Match User sftp_* ChrootDirectory {servers_dir}/%u ForceCommand internal-sftp -d /data PasswordAuthentication yes PermitTunnel no AllowAgentForwarding no AllowTcpForwarding no X11Forwarding no """ # Write via temp file to avoid shell escaping issues import tempfile with tempfile.NamedTemporaryFile(mode='w', suffix='.conf', delete=False) as tf: tf.write(sftp_block) tf_path = tf.name run_cmd(f"cat {tf_path} >> {sshd_config}") run_cmd(f"rm -f {tf_path}") # Restart sshd run_cmd("systemctl restart ssh 2>/dev/null || systemctl restart sshd 2>/dev/null || /usr/sbin/sshd") logger.info("SFTP infrastructure configured") def _apply_sftp_acl(username, data_dir): """Apply POSIX ACLs so both SFTP user and container user (UID 1000) can read/write all files.""" # Change ownership of all files to SFTP user so they can chmod/utime after upload run_cmd(f"chown -R {username}:nogroup {data_dir}") # Grant SFTP user full access to all existing files owned by container (1000) run_cmd(f"setfacl -R -m u:{username}:rwX {data_dir}") # Default ACL: new files created by container will also be accessible by SFTP user run_cmd(f"setfacl -R -d -m u:{username}:rwX {data_dir}") # Grant container user (1000) access to files created by SFTP user run_cmd(f"setfacl -R -m u:1000:rwX {data_dir}") # Default ACL: new files created by SFTP user will also be accessible by container run_cmd(f"setfacl -R -d -m u:1000:rwX {data_dir}") def create_sftp_user(uuid): """Create a restricted SFTP user for a game server.""" try: # Ensure SSH/SFTP infrastructure is ready ensure_sftp_ready() short = uuid[:12].replace("-", "") username = f"sftp_{short}" data_dir = Path(DATA_DIR) / "servers" / uuid / "data" if not data_dir.exists(): data_dir.mkdir(parents=True, exist_ok=True) # Generate random password import secrets import string password = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(16)) # ChrootDirectory requires: chroot dir owned by root, writable subdir for user # Structure: /servers// (root:root 755) -> /servers//data/ (sftp_user:nogroup) # sshd ChrootDirectory = /servers/%u -> symlink sftp_username -> uuid dir server_dir = Path(DATA_DIR) / "servers" / uuid server_dir.mkdir(parents=True, exist_ok=True) chroot_link = Path(DATA_DIR) / "servers" / username # Create symlink: /servers/sftp_xxxx -> /servers/ if chroot_link.is_symlink() or chroot_link.exists(): run_cmd(f"rm -f {chroot_link}") run_cmd(f"ln -s {server_dir} {chroot_link}") # Chroot dir must be owned by root run_cmd(f"chown root:root {server_dir}") run_cmd(f"chmod 755 {server_dir}") # Check if user already exists rc, out, _ = run_cmd(f"id {username}") if rc == 0: # User exists - fix shell and home dir (home must be chroot-relative) run_cmd(f"usermod -s /bin/false -d /data {username}") # Reset password rc2, _, err2 = run_cmd(f"echo '{username}:{password}' | chpasswd") if rc2 != 0: return {"ok": False, "error": f"Failed to reset password: {err2}"} # Ensure data dir writable by user run_cmd(f"chown {username}:nogroup {data_dir}") # ACLs: grant SFTP user access to container files (1000) and vice versa _apply_sftp_acl(username, data_dir) return {"ok": True, "username": username, "password": password, "existed": True} # Create system user with /bin/false shell (ForceCommand internal-sftp bypasses shell) # Home dir = /data (chroot-relative) so SSH resolves it correctly after chroot rc, _, err = run_cmd( f"useradd -M -d /data -s /bin/false -g nogroup {username}" ) if rc != 0: return {"ok": False, "error": f"Failed to create user: {err}"} # Set password rc2, _, err2 = run_cmd(f"echo '{username}:{password}' | chpasswd") if rc2 != 0: return {"ok": False, "error": f"Failed to set password: {err2}"} # Set ownership: data dir writable by SFTP user run_cmd(f"chown {username}:nogroup {data_dir}") # ACLs: grant SFTP user access to container files (1000) and vice versa _apply_sftp_acl(username, data_dir) logger.info(f"SFTP user created: {username} for server {uuid}") return {"ok": True, "username": username, "password": password, "existed": False} except Exception as e: return {"ok": False, "error": str(e)} def reset_sftp_password(uuid): """Reset SFTP password for a game server.""" try: short = uuid[:12].replace("-", "") username = f"sftp_{short}" rc, _, _ = run_cmd(f"id {username}") if rc != 0: return {"ok": False, "error": "SFTP user does not exist"} import secrets import string password = ''.join(secrets.choice(string.ascii_letters + string.digits) for _ in range(16)) rc2, _, err2 = run_cmd(f"echo '{username}:{password}' | chpasswd") if rc2 != 0: return {"ok": False, "error": f"Failed to reset password: {err2}"} logger.info(f"SFTP password reset for {username}") return {"ok": True, "username": username, "password": password} except Exception as e: return {"ok": False, "error": str(e)} def delete_sftp_user(uuid): """Delete SFTP user for a game server.""" try: short = uuid[:12].replace("-", "") username = f"sftp_{short}" rc, _, _ = run_cmd(f"id {username}") if rc != 0: return {"ok": True, "message": "User does not exist"} run_cmd(f"userdel {username}") logger.info(f"SFTP user deleted: {username}") return {"ok": True} except Exception as e: return {"ok": False, "error": str(e)} def get_sftp_status(uuid): """Check if SFTP user exists for a server.""" try: short = uuid[:12].replace("-", "") username = f"sftp_{short}" rc, _, _ = run_cmd(f"id {username}") return {"ok": True, "exists": rc == 0, "username": username if rc == 0 else None} except Exception as e: return {"ok": False, "error": str(e)} class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): """HTTPServer with threading support and configurable thread limits. IMPORTANT: Limits concurrent connections to prevent thread exhaustion. SSE connections stay open, so we need to cap them to avoid daemon hanging. """ daemon_threads = True max_threads = 50 # Max concurrent request threads (SSE, file ops, etc.) active_connections = 0 connection_lock = threading.Lock() def process_request(self, request, client_address): """Override to enforce thread limit before accepting connection.""" with self.connection_lock: if self.active_connections >= self.max_threads: logger.warning(f"Thread limit reached ({self.max_threads}), rejecting new connection from {client_address}") try: request.close() except Exception: pass return self.active_connections += 1 try: super().process_request(request, client_address) finally: with self.connection_lock: self.active_connections = max(0, self.active_connections - 1) def shutdown_request(self, request): """Override to properly close SSL connections.""" try: request.unwrap() except Exception: pass try: super().shutdown_request(request) except Exception: pass class AgentRequestHandler(BaseHTTPRequestHandler): """HTTP request handler for the Game Agent API.""" daemon_token = "" panel_url = "" protocol_version = "HTTP/1.1" def handle(self): """Override to catch SSL errors that cause 'unexpected eof' on clients.""" try: super().handle() except ssl.SSLEOFError: pass except ssl.SSLError as e: if 'EOF' in str(e): pass else: logger.warning(f"SSL error: {e}") except (ConnectionResetError, BrokenPipeError, OSError): pass def log_message(self, format, *args): logger.info(f"{self.client_address[0]} - {format % args}") def send_json(self, data, status=200): body = json.dumps(data).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.send_header("Connection", "close") self.send_header("X-Agent-Version", AGENT_VERSION) self.end_headers() self.wfile.write(body) def authenticate(self): """Verify the Bearer token from the Authorization header.""" auth_header = self.headers.get("Authorization", "") if not auth_header.startswith("Bearer "): self.send_json({"error": "Unauthorized"}, 401) return False token = auth_header[7:] if not hmac.compare_digest(token, self.daemon_token): self.send_json({"error": "Invalid token"}, 403) return False return True def read_body(self): """Read and parse the JSON request body.""" content_length = int(self.headers.get("Content-Length", 0)) if content_length == 0: return {} body = self.rfile.read(content_length) try: return json.loads(body) except json.JSONDecodeError: return {} def parse_path(self): """Parse the URL path, returning (path, query_params).""" parsed = urlparse(self.path) return parsed.path, parse_qs(parsed.query) def do_GET(self): path, params = self.parse_path() if not self.authenticate(): return # GET /api/status - Node status and metrics if path == "/api/status": metrics = get_system_metrics() self.send_json(metrics) return # GET /api/servers/{uuid}/logs - Console logs match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/logs$", path) if match: uuid = match.group(1) result = get_console_logs(uuid) self.send_json(result) return # GET /api/servers/{uuid}/console-stream - SSE console stream match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/console-stream$", path) if match: uuid = match.group(1) stream_console_logs(self, uuid) return # GET /api/servers/{uuid}/stats - Container resource usage match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/stats$", path) if match: uuid = match.group(1) result = get_container_stats(uuid) self.send_json(result) return # GET /api/servers/{uuid}/install-status - Poll install progress match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/install-status$", path) if match: uuid = match.group(1) result = _read_install_status(uuid) self.send_json({"ok": True, **result}) return # GET /api/servers/{uuid}/sftp/status - Check SFTP user status match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/sftp/status$", path) if match: uuid = match.group(1) result = get_sftp_status(uuid) self.send_json(result) return # GET /api/servers/{uuid}/files/list - List files match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/list$", path) if match: uuid = match.group(1) file_path = params.get("path", ["/"])[0] result = list_files(uuid, file_path) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # GET /api/servers/{uuid}/files/read - Read file match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/read$", path) if match: uuid = match.group(1) file_path = params.get("path", [""])[0] result = read_file_content(uuid, file_path) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # GET /api/servers/{uuid}/files/download - Get download URL match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/download$", path) if match: uuid = match.group(1) file_path = params.get("path", [""])[0] result = get_file_download_url(uuid, file_path) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # GET /api/download/{token} - Serve file download match = re.match(r"^/api/download/([a-zA-Z0-9_-]+)$", path) if match: token = match.group(1) dl = _download_tokens.pop(token, None) if not dl or time.time() - dl["created"] > 300: self.send_json({"ok": False, "error": "Invalid or expired token"}, 404) return file_path = Path(dl["path"]) if not file_path.exists() or not file_path.is_file(): self.send_json({"ok": False, "error": "File not found"}, 404) return try: file_size = file_path.stat().st_size self.send_response(200) self.send_header("Content-Type", "application/octet-stream") self.send_header("Content-Disposition", f'attachment; filename="{file_path.name}"') self.send_header("Content-Length", str(file_size)) self.send_header("Connection", "close") self.end_headers() with open(str(file_path), 'rb') as f: while True: chunk = f.read(65536) if not chunk: break self.wfile.write(chunk) except Exception as e: logger.error(f"Download failed: {e}") return # GET /api/servers/{uuid}/backups/list - List backups match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/backups/list$", path) if match: uuid = match.group(1) result = list_backups(uuid) self.send_json(result) return # GET /api/servers/{uuid}/fraud-scan - Scan for abuse/fraud match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/fraud-scan$", path) if match: uuid = match.group(1) name = get_container_name(uuid) status = docker_server_status(uuid) if status != "running": self.send_json({"ok": True, "threats": [], "risk_level": "clean", "note": "Server not running, limited scan"}) return result = FraudDetector.scan_container(uuid, name) server_path = Path(DATA_DIR) / "servers" / uuid was_critical = _handle_fraud_detection( uuid, name, server_path, result, panel_url=self.panel_url, token=self.daemon_token, ) if was_critical: result["auto_suspended"] = True result["suspend_reason"] = _build_fraud_suspend_reason(result["threats"]) self.send_json(result) return self.send_json({"error": "Not found"}, 404) def do_POST(self): path, params = self.parse_path() if not self.authenticate(): return data = self.read_body() # POST /api/servers - Install new server if path == "/api/servers": result = install_server(data) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/reinstall - Reinstall server match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/reinstall$", path) if match: uuid = match.group(1) result = reinstall_server(uuid, data) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/power - Power action match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/power$", path) if match: uuid = match.group(1) action = data.get("action", "") # Panel may send sync config (startup_command, environment) for wrapper rebuild sync_config = {} if data.get("startup_command"): sync_config["startup_command"] = data["startup_command"] if data.get("environment"): sync_config["environment"] = data["environment"] # For start/restart: validate synchronously, then run Docker command # in background thread and respond immediately so PHP doesn't timeout if action in ("start", "restart"): # Block if fraud-suspended if uuid in _fraud_suspended_servers: self.send_json({ "ok": False, "error": "Serwer zawieszony za oszustwo - skontaktuj się z pomocą techniczną" }, 403) return name_check = get_container_name(uuid) check_status = docker_server_status(uuid) if action == "start" and check_status == "not_found": self.send_json({"ok": False, "error": "Server not installed"}, 400) return # Respond immediately - Docker start/restart runs in background self.send_json({"ok": True, "message": f"Action '{action}' accepted, processing..."}) def _bg_power(): try: result = power_action(uuid, action, sync_config=sync_config if sync_config else None) if not result.get("ok"): logger.warning(f"Background power action failed for {uuid}: {result.get('error') or result.get('message')}") else: # Check if container crashed immediately after start (within 5s) time.sleep(5) post_status = docker_server_status(uuid) if post_status in ("exited", "dead"): name_for_log = get_container_name(uuid) rc_log, logs, _ = run_cmd(f"docker logs --tail 20 {name_for_log}", timeout=5) logger.warning(f"Server {uuid} crashed immediately after {action}. Container status: {post_status}. Last logs:\n{logs}") except Exception as e: logger.error(f"Background power action failed for {uuid}: {e}") threading.Thread(target=_bg_power, daemon=True).start() return # stop/kill run synchronously (they are fast) result = power_action(uuid, action, sync_config=sync_config if sync_config else None) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/unsuspend - Clear fraud suspension match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/unsuspend$", path) if match: uuid = match.group(1) _fraud_suspended_servers.discard(uuid) logger.info(f"Fraud suspension cleared for {uuid} (admin unsuspend)") self.send_json({"ok": True, "message": "Fraud suspension cleared"}) return # POST /api/servers/{uuid}/command - Console command match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/command$", path) if match: uuid = match.group(1) command = data.get("command", "") result = send_command(uuid, command) self.send_json(result) return # POST /api/servers/{uuid}/auto-restart - Update auto-restart config match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/auto-restart$", path) if match: uuid = match.group(1) server_dir = Path(DATA_DIR) / "servers" / uuid config_file = server_dir / "config.json" if not config_file.exists(): self.send_json({"ok": False, "error": "Server not found"}, 404) return try: with open(config_file) as f: cfg = json.load(f) cfg["auto_restart"] = { "enabled": bool(data.get("enabled", False)), "interval_hours": data.get("interval_hours"), "on_crash": bool(data.get("on_crash", True)), "max_crashes": int(data.get("max_crashes", 3)), "crash_window": int(data.get("crash_window", 3600)), } with open(config_file, "w") as f: json.dump(cfg, f, indent=2) logger.info(f"[{uuid}] Auto-restart config updated: {cfg['auto_restart']}") self.send_json({"ok": True, "message": "Auto-restart config updated"}) except Exception as e: self.send_json({"ok": False, "error": str(e)}, 500) return # POST /api/servers/{uuid}/files/write - Write file match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/write$", path) if match: uuid = match.group(1) file_path = data.get("path", "") content = data.get("content", "") encoding = data.get("encoding", None) result = write_file_content(uuid, file_path, content, encoding) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/create-folder - Create folder match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/create-folder$", path) if match: uuid = match.group(1) file_path = data.get("path", "") result = create_directory(uuid, file_path) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/delete - Delete file/folder match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/delete$", path) if match: uuid = match.group(1) file_path = data.get("path", "") result = delete_path(uuid, file_path) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/rename - Rename file/folder match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/rename$", path) if match: uuid = match.group(1) file_path = data.get("path", "") new_name = data.get("new_name", "") if not new_name: self.send_json({"ok": False, "error": "New name required"}, 400) return result = rename_path(uuid, file_path, new_name) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/move - Move file/folder match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/move$", path) if match: uuid = match.group(1) source = data.get("source", "") destination = data.get("destination", "") if not source or not destination and destination != '/': self.send_json({"ok": False, "error": "Source and destination required"}, 400) return result = move_path(uuid, source, destination) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/bulk-delete - Delete multiple files match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/bulk-delete$", path) if match: uuid = match.group(1) root = data.get("root", "/") files = data.get("files", []) if not files or not isinstance(files, list): self.send_json({"ok": False, "error": "Files list required"}, 400) return result = bulk_delete(uuid, root, files) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/bulk-move - Move multiple files match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/bulk-move$", path) if match: uuid = match.group(1) root = data.get("root", "/") files = data.get("files", []) destination = data.get("destination", "") if not files or not isinstance(files, list): self.send_json({"ok": False, "error": "Files list required"}, 400) return if not destination and destination != '/': self.send_json({"ok": False, "error": "Destination required"}, 400) return result = bulk_move(uuid, root, files, destination) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/compress - Compress files match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/compress$", path) if match: uuid = match.group(1) root = data.get("root", "/") files = data.get("files", []) if not files or not isinstance(files, list): self.send_json({"ok": False, "error": "Files list required"}, 400) return result = compress_files(uuid, root, files) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/files/decompress - Decompress archive match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/files/decompress$", path) if match: uuid = match.group(1) root = data.get("root", "/") file = data.get("file", "") if not file: self.send_json({"ok": False, "error": "File name required"}, 400) return result = decompress_file(uuid, root, file) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/sftp/create - Create SFTP user match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/sftp/create$", path) if match: uuid = match.group(1) result = create_sftp_user(uuid) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/sftp/reset-password - Reset SFTP password match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/sftp/reset-password$", path) if match: uuid = match.group(1) result = reset_sftp_password(uuid) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/backups/create - Create backup match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/backups/create$", path) if match: uuid = match.group(1) backup_uuid = data.get("backup_uuid", "") backup_name = data.get("name", "") if not backup_uuid: self.send_json({"ok": False, "error": "Missing backup_uuid"}, 400) return result = create_backup(uuid, backup_uuid, backup_name) status = 200 if result.get("ok") else 400 self.send_json(result, status) return # POST /api/servers/{uuid}/backups/restore - Restore backup match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/backups/restore$", path) if match: uuid = match.group(1) backup_uuid = data.get("backup_uuid", "") if not backup_uuid: self.send_json({"ok": False, "error": "Missing backup_uuid"}, 400) return result = restore_backup(uuid, backup_uuid) status = 200 if result.get("ok") else 400 self.send_json(result, status) return self.send_json({"error": "Not found"}, 404) def do_DELETE(self): path, params = self.parse_path() if not self.authenticate(): return # DELETE /api/servers/{uuid}/sftp - Delete SFTP user match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/sftp$", path) if match: uuid = match.group(1) result = delete_sftp_user(uuid) self.send_json(result) return # DELETE /api/servers/{uuid} - Delete server match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)$", path) if match: uuid = match.group(1) result = delete_server(uuid) self.send_json(result) return # DELETE /api/servers/{uuid}/backups/{backup_uuid} - Delete backup match = re.match(r"^/api/servers/([a-zA-Z0-9_-]+)/backups/([a-zA-Z0-9_-]+)$", path) if match: uuid = match.group(1) backup_uuid = match.group(2) result = delete_backup(uuid, backup_uuid) self.send_json(result) return self.send_json({"error": "Not found"}, 404) def run_heartbeat(panel_url, token, interval=45): """Background thread that sends heartbeat to the panel every N seconds.""" if not panel_url: return import urllib.request import urllib.error # Disable SSL verification (self-signed certs on panels) ctx = _get_panel_ssl_context() consecutive_failures = 0 max_retries = 3 heartbeat_count = 0 while True: success = False for attempt in range(max_retries): try: metrics = get_system_metrics() metrics["token"] = token data = json.dumps(metrics).encode("utf-8") req = urllib.request.Request( panel_url, data=data, headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(req, timeout=15, context=ctx) as resp: logger.debug(f"Heartbeat sent, status: {resp.status}") consecutive_failures = 0 success = True break except Exception as e: wait = min(5 * (attempt + 1), 15) logger.warning(f"Heartbeat attempt {attempt + 1}/{max_retries} failed: {e} (retry in {wait}s)") if attempt < max_retries - 1: time.sleep(wait) if not success: consecutive_failures += 1 logger.error(f"Heartbeat failed after {max_retries} attempts (consecutive failures: {consecutive_failures})") # Check for auto-update every ~40 heartbeats (~30 min at 45s interval) heartbeat_count += 1 if heartbeat_count % 40 == 0: try: _auto_update(panel_url) except Exception as e: logger.debug(f"Periodic auto-update check failed: {e}") time.sleep(interval) # --- Background Self-Healing Monitor ----------------------------- # Fraud scan state: track which servers were already suspended to avoid spamming _fraud_suspended_servers = set() _fraud_scan_cycle = 0 # Counter for fraud scan frequency def _report_fraud_to_panel(panel_url, token, uuid, fraud_result): """Report critical fraud detection to the panel for auto-suspension. Calls POST /api/internal/fraud-report/{uuid} on the panel to: 1. Mark the server as suspended in the database 2. Record the specific reason (what was detected) 3. Create audit log entry Returns True if the panel confirmed suspension. """ if not panel_url: return False import urllib.request import urllib.error # Derive base panel URL from heartbeat URL base_url = panel_url.rstrip("/") for suffix in ["/api/game-node-heartbeat", "/game-node-heartbeat"]: if base_url.endswith(suffix): base_url = base_url[: -len(suffix)] break report_url = f"{base_url}/api/internal/fraud-report/{uuid}" payload = { "token": token, "risk_level": fraud_result.get("risk_level", "critical"), "threats": fraud_result.get("threats", []), "scanned_at": fraud_result.get("scanned_at"), } ctx = _get_panel_ssl_context() try: data = json.dumps(payload).encode("utf-8") req = urllib.request.Request( report_url, data=data, headers={"Content-Type": "application/json"}, method="POST", ) logger.info(f"[{uuid}] Reporting fraud to panel: {report_url}") with urllib.request.urlopen(req, timeout=15, context=ctx) as resp: body = json.loads(resp.read().decode("utf-8", errors="replace")) if body.get("ok") and body.get("suspended"): logger.warning( f"[{uuid}] Panel confirmed suspension: {body.get('reason', 'N/A')}" ) return True elif body.get("already"): logger.info(f"[{uuid}] Server already suspended on panel") return True else: logger.warning(f"[{uuid}] Panel fraud-report unexpected response: {body}") except Exception as e: logger.warning(f"[{uuid}] Failed to report fraud to panel ({report_url}): {e}") return False def _build_fraud_suspend_reason(threats): """Build a human-readable suspend reason from detected threats.""" reason_map = { "malicious_process": "złośliwy proces", "crypto_mining_cpu_pattern": "kopanie kryptowalut (CPU)", "miner_config": "konfiguracja minera", "malicious_file": "złośliwe pliki", "ai_model_file": "pliki modeli AI", "ai_model_directory": "katalog modeli AI", "ai_workload": "obciążenie AI/ML", "ai_pip_packages": "pakiety AI", "ai_gpu_access": "dostęp GPU", "ai_inference_port": "port serwera AI", "ai_memory_abuse": "nadużycie pamięci RAM", "mining_pool_connection": "połączenie z mining pool", "suspicious_port": "podejrzany port", "hidden_miner": "ukryty miner", "deleted_binary": "usunięty plik binarny", "ld_preload_rootkit": "rootkit LD_PRELOAD", "kernel_thread_masquerade": "maskowanie procesu", "ddos_connection_flood": "flood połączeń (DDoS)", "ddos_syn_flood": "SYN flood (DDoS)", "ddos_fan_out": "fan-out (DDoS)", "ddos_single_target_flood": "flood na cel (DDoS)", "ddos_udp_amplification": "UDP amplification (DDoS)", "ddos_raw_socket": "RAW socket (DDoS)", "ddos_bandwidth_abuse": "nadużycie łącza (DDoS)", "ddos_icmp_flood": "ICMP flood (DDoS)", "ddos_firewall_manipulation": "manipulacja firewallem", "ddos_attack_script": "skrypt ataku DDoS", "ddos_attack_process": "narzędzie DDoS", "ddos_conntrack_overflow": "przepełnienie conntrack", "unknown_binary": "nieznany plik ELF", } threat_types = list({t.get("type", "unknown") for t in threats}) reasons = [reason_map.get(t, t) for t in threat_types] return ", ".join(reasons[:5]) def _handle_fraud_detection(uuid, name, server_path, fraud_result, panel_url=None, token=None): """Handle a fraud scan result - stop container, write report, report to panel. Called from both the dedicated fraud scanner and the on-demand API endpoint. """ if fraud_result.get("risk_level") == "critical": threat_count = len(fraud_result.get("threats", [])) threat_reason = _build_fraud_suspend_reason(fraud_result["threats"]) logger.warning( f"[{uuid}] ⚠️ FRAUD DETECTED: {threat_count} critical threats! " f"Powód: {threat_reason}" ) # 1. Immediately kill the container (no grace period for fraud) logger.warning(f"[{uuid}] Auto-killing server due to fraud detection") run_cmd(f"docker kill {name}", timeout=15) # Also docker stop as fallback run_cmd(f"docker stop -t 0 {name}", timeout=10) # 2. Write local fraud report report_file = server_path / "fraud_report.json" try: fraud_result["auto_suspended"] = True fraud_result["suspend_reason"] = threat_reason fraud_result["suspended_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) with open(report_file, "w") as rf: json.dump(fraud_result, rf, indent=2) except Exception: pass # 3. Report to panel for database suspension if panel_url and token: suspended = _report_fraud_to_panel(panel_url, token, uuid, fraud_result) if suspended: _fraud_suspended_servers.add(uuid) logger.warning( f"[{uuid}] ✅ Server SUSPENDED on panel - powód: {threat_reason}" ) else: # Panel report failed - retry in next cycle, but still block locally _fraud_suspended_servers.add(uuid) logger.warning( f"[{uuid}] ⚠️ Could not report to panel, server killed locally. " f"Will retry on next scan cycle." ) else: _fraud_suspended_servers.add(uuid) logger.warning( f"[{uuid}] Server killed (no panel URL to report suspension)" ) return True # Was critical elif fraud_result.get("risk_level") == "suspicious": logger.info( f"[{uuid}] Fraud scan: suspicious activity - " f"{[t['description'] for t in fraud_result['threats']]}" ) # Write suspicion report (don't suspend yet) report_file = server_path / "fraud_report.json" try: fraud_result["auto_suspended"] = False with open(report_file, "w") as rf: json.dump(fraud_result, rf, indent=2) except Exception: pass return False # Not critical # Queue for servers that need an immediate fraud scan (e.g. just started) _fraud_scan_queue = [] _fraud_scan_queue_lock = threading.Lock() def _queue_fraud_scan(uuid, delay=30): """Queue a server for fraud scanning after a delay (e.g. after start). The dedicated fraud thread picks these up. """ scan_at = time.time() + delay with _fraud_scan_queue_lock: # Don't double-queue if not any(item[0] == uuid for item in _fraud_scan_queue): _fraud_scan_queue.append((uuid, scan_at)) logger.info(f"[{uuid}] Queued for fraud scan in {delay}s") def _background_fraud_scanner(interval=60, panel_url=None, token=None): """Dedicated fraud scanning thread - runs every 60 seconds. Much more aggressive than the self-healing loop (5 min). Scans all running servers for fraud indicators and takes immediate action. Also processes the priority scan queue (servers just started). """ # Wait 30s after agent start before first scan time.sleep(30) logger.info("🔍 Fraud scanner started - scanning every 60s") cycle_count = 0 while True: cycle_count += 1 cycle_start = time.time() servers_scanned = 0 threats_found = 0 try: # 1. Process priority scan queue first (recently started servers) now = time.time() queued_uuids = [] with _fraud_scan_queue_lock: ready = [(u, t) for u, t in _fraud_scan_queue if t <= now] for item in ready: _fraud_scan_queue.remove(item) queued_uuids.append(item[0]) for uuid in queued_uuids: if uuid in _fraud_suspended_servers: continue name = get_container_name(uuid) status = docker_server_status(uuid) if status != "running": continue server_path = Path(DATA_DIR) / "servers" / uuid try: logger.info(f"[{uuid}] Priority fraud scan (post-start)") scan_start = time.time() fraud_result = FraudDetector.scan_container(uuid, name) scan_time = time.time() - scan_start risk = fraud_result.get("risk_level", "unknown") threat_count = len(fraud_result.get("threats", [])) logger.info(f"[{uuid}] Priority scan done in {scan_time:.1f}s - {risk} ({threat_count} threats)") _handle_fraud_detection(uuid, name, server_path, fraud_result, panel_url, token) except Exception as e: logger.warning(f"[{uuid}] Priority fraud scan error: {e}") # 2. Scan ALL running servers servers_dir = Path(DATA_DIR) / "servers" if not servers_dir.exists(): logger.info(f"Fraud scanner cycle #{cycle_count}: servers dir not found, sleeping") time.sleep(interval) continue for server_path in servers_dir.iterdir(): if not server_path.is_dir() or server_path.name.startswith("sftp_"): continue uuid = server_path.name # Skip already suspended if uuid in _fraud_suspended_servers: continue name = get_container_name(uuid) status = docker_server_status(uuid) # For stopped/exited containers: still do file-only scan # (malicious files persist on disk even after container stops) if status != "running": try: file_threats = FraudDetector._scan_files(uuid) if any(t.get("severity") == "critical" for t in file_threats): fraud_result = { "ok": True, "threats": file_threats, "risk_level": "critical", "scanned_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } servers_scanned += 1 threats_found += len(file_threats) logger.warning( f"[{uuid}] File scan on stopped container found {len(file_threats)} threats - " f"{[t['description'] for t in file_threats[:3]]}" ) _handle_fraud_detection(uuid, name, server_path, fraud_result, panel_url, token) except Exception as e: logger.warning(f"[{uuid}] File scan (stopped) error: {e}") continue try: scan_start = time.time() fraud_result = FraudDetector.scan_container(uuid, name) scan_time = time.time() - scan_start risk = fraud_result.get("risk_level", "unknown") threat_count = len(fraud_result.get("threats", [])) servers_scanned += 1 threats_found += threat_count if risk != "clean": logger.info( f"[{uuid}] Fraud scan: {risk} ({threat_count} threats) in {scan_time:.1f}s - " f"{[t['description'] for t in fraud_result.get('threats', [])[:3]]}" ) _handle_fraud_detection(uuid, name, server_path, fraud_result, panel_url, token) except Exception as e: logger.warning(f"[{uuid}] Fraud scan error: {e}") except Exception as e: logger.warning(f"Fraud scanner loop error: {e}") cycle_time = time.time() - cycle_start logger.info( f"🔍 Fraud scanner cycle #{cycle_count}: " f"scanned {servers_scanned} servers in {cycle_time:.1f}s, " f"{threats_found} total threats" ) time.sleep(interval) # Track scheduled restart times and crash counts {uuid: {"next_restart": float, "crashes": [(timestamp, ...)]}} _auto_restart_state = {} def _background_self_healing_loop(interval=300, panel_url=None, token=None): """Background thread that periodically checks running servers for issues. Every `interval` seconds: - Re-applies port config (catches LinuxGSM port resets) - Checks for crashed containers and attempts auto-restart (watchdog) - Handles scheduled periodic restarts (every X hours) - Validates file permissions NOTE: Fraud scanning is handled by dedicated _background_fraud_scanner thread (60s). """ while True: time.sleep(interval) try: servers_dir = Path(DATA_DIR) / "servers" if not servers_dir.exists(): continue for server_path in servers_dir.iterdir(): if not server_path.is_dir() or server_path.name.startswith("sftp_"): continue uuid = server_path.name config_file = server_path / "config.json" if not config_file.exists(): continue try: with open(config_file) as f: cfg = json.load(f) except Exception: continue name = get_container_name(uuid) status = docker_server_status(uuid) port = cfg.get("port") data_dir = server_path / "data" startup_cmd = cfg.get("startup_command", "") docker_image = cfg.get("docker_image", "") environment = cfg.get("environment", {}) ar = cfg.get("auto_restart", {}) if status == "running": # -- Running server: ensure port config is correct -- if port: _patch_server_port(data_dir, port or _get_default_port(startup_cmd, docker_image, environment), startup_cmd, docker_image) # LinuxGSM: re-apply port config periodically lgsm_game = environment.get("LGSM_GAMESERVER", "") if lgsm_game: safe_lgsm = re.sub(r'[^a-zA-Z0-9_-]', '', lgsm_game) _write_linuxgsm_port_config(uuid, name, safe_lgsm, port, server_path) # -- Scheduled periodic restart -- if ar.get("enabled") and ar.get("interval_hours"): interval_hours = ar["interval_hours"] state = _auto_restart_state.setdefault(uuid, {}) now = time.time() if "next_restart" not in state: # Initialize: next restart = now + interval state["next_restart"] = now + (interval_hours * 3600) elif now >= state["next_restart"]: logger.info(f"[{uuid}] Scheduled restart (every {interval_hours}h)") rc_stop, _, err_stop = run_cmd(f"docker stop -t 10 {name}", timeout=25) if rc_stop != 0: run_cmd(f"docker kill {name}", timeout=10) if port: _kill_time_wait_sockets(port) _wait_for_port_free(port, timeout=30) rc, _, _ = run_cmd(f"docker start {name}", timeout=30) if rc == 0: logger.info(f"[{uuid}] Scheduled restart completed successfully") else: logger.warning(f"[{uuid}] Scheduled restart failed") state["next_restart"] = time.time() + (interval_hours * 3600) elif status == "exited": # -- Crashed server: check install status -- install_status = _read_install_status(uuid) # Do NOT restart fraud-suspended servers if uuid in _fraud_suspended_servers: pass elif install_status.get("status") == "stopped": pass # Intentionally stopped - do nothing elif (install_status.get("status") == "done" and install_status.get("healthy")): # Check auto-restart on crash settings on_crash = ar.get("on_crash", True) # Default: auto-restart on crash max_crashes = ar.get("max_crashes", 3) crash_window = ar.get("crash_window", 3600) if not on_crash: logger.info(f"[{uuid}] Server crashed but auto_restart_on_crash is disabled") continue # Track crash count within window state = _auto_restart_state.setdefault(uuid, {}) now = time.time() crashes = state.get("crashes", []) # Remove old crashes outside window crashes = [t for t in crashes if now - t < crash_window] crashes.append(now) state["crashes"] = crashes if len(crashes) > max_crashes: logger.warning( f"[{uuid}] Server crashed {len(crashes)} times in {crash_window}s " f"(max={max_crashes}) - giving up auto-restart" ) _write_install_status(uuid, "crashed", f"Serwer crashuje zbyt często ({len(crashes)}x w ciągu " f"{crash_window // 60} min). Automatyczny restart wyłączony.") continue logger.info(f"[{uuid}] Crash watchdog: attempt {len(crashes)}/{max_crashes}, restarting...") heal = _self_healing_check(uuid, name) if port: _kill_time_wait_sockets(port) _wait_for_port_free(port, timeout=30) rc, _, _ = run_cmd(f"docker start {name}", timeout=30) if rc == 0: logger.info(f"[{uuid}] Crash watchdog: restarted successfully (crash #{len(crashes)})") else: logger.warning(f"[{uuid}] Crash watchdog: restart failed, actions={heal.get('actions', [])}") except Exception as e: logger.debug(f"Self-healing loop error: {e}") def _auto_update(panel_url): """ Check the panel for a newer agent version and self-update if available. Downloads the latest game_agent.py, replaces the current script, and restarts. """ if not panel_url: return False import urllib.request import urllib.error ctx = _get_panel_ssl_context() try: # Strip heartbeat path to get base panel URL base_url = panel_url.rstrip("/") for suffix in ["/api/game-node-heartbeat", "/game-node-heartbeat"]: if base_url.endswith(suffix): base_url = base_url[: -len(suffix)] break download_url = f"{base_url}/game-agent/download-script" logger.info(f"Auto-update: checking {download_url}") req = urllib.request.Request(download_url, method="GET") with urllib.request.urlopen(req, timeout=30, context=ctx) as resp: new_code = resp.read().decode("utf-8", errors="replace") # Extract version from downloaded script import re as _re m = _re.search(r'AGENT_VERSION\s*=\s*["\']([^"\']+)["\']', new_code) if not m: logger.debug("Auto-update: could not detect version in downloaded script") return False remote_version = m.group(1) if remote_version == AGENT_VERSION: logger.debug(f"Auto-update: already on latest version ({AGENT_VERSION})") return False # Compare versions (simple tuple comparison) def ver_tuple(v): return tuple(int(x) for x in v.split(".") if x.isdigit()) if ver_tuple(remote_version) <= ver_tuple(AGENT_VERSION): logger.debug(f"Auto-update: remote {remote_version} <= local {AGENT_VERSION}, skip") return False # Write new agent script current_script = os.path.abspath(__file__) backup_path = current_script + ".bak" logger.info(f"Auto-update: upgrading {AGENT_VERSION} → {remote_version}") # Backup current import shutil shutil.copy2(current_script, backup_path) # Write new version with open(current_script, "w", newline="\n") as f: f.write(new_code) logger.info(f"Auto-update: wrote new version to {current_script}") logger.info("Auto-update: restarting agent via systemd...") # Restart via systemd (the service will pick up the new script) os.system("systemctl restart game-agent 2>/dev/null &") return True except Exception as e: logger.warning(f"Auto-update failed: {e}") return False def main(): global DATA_DIR, VERIFY_PANEL_SSL, PANEL_CA_BUNDLE parser = argparse.ArgumentParser(description="IkaByte Game Agent") parser.add_argument("--token", required=True, help="Daemon authentication token") parser.add_argument("--port", type=int, default=8443, help="Port to listen on (default: 8443)") parser.add_argument("--host", default="0.0.0.0", help="Host to bind to (default: 0.0.0.0)") parser.add_argument("--panel-url", default="", help="Panel URL for heartbeat (e.g. https://panel.example.com/api/game-node-heartbeat)") parser.add_argument("--data-dir", default=DATA_DIR, help="Data directory (default: /var/lib/game-agent)") parser.add_argument("--ssl-cert", default="", help="Path to SSL certificate") parser.add_argument("--ssl-key", default="", help="Path to SSL private key") parser.add_argument("--no-ssl", action="store_true", help="Disable SSL (not recommended for production)") parser.add_argument("--verify-panel-ssl", action="store_true", default=False, help="Verify panel SSL certificates (recommended for production)") parser.add_argument("--ca-bundle", default="", help="Path to CA certificate bundle for verifying panel SSL") args = parser.parse_args() DATA_DIR = args.data_dir VERIFY_PANEL_SSL = args.verify_panel_ssl PANEL_CA_BUNDLE = args.ca_bundle # Create data directories Path(DATA_DIR).mkdir(parents=True, exist_ok=True) Path(f"{DATA_DIR}/servers").mkdir(parents=True, exist_ok=True) Path(LOG_DIR).mkdir(parents=True, exist_ok=True) # Add file logging file_handler = logging.FileHandler(f"{LOG_DIR}/agent.log") file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) logger.addHandler(file_handler) # Check Docker rc, _, _ = run_cmd("docker info >/dev/null 2>&1") if rc != 0: logger.error("Docker is not running or not installed! Game servers require Docker.") logger.error("Install Docker: curl -fsSL https://get.docker.com | sh") else: logger.info("Docker is available") # Set token and panel URL for handler AgentRequestHandler.daemon_token = args.token AgentRequestHandler.panel_url = args.panel_url or "" # Start heartbeat thread if args.panel_url: # Auto-update check on startup try: if _auto_update(args.panel_url): logger.info("Auto-update initiated, agent will restart...") return # systemd will restart us except Exception as e: logger.warning(f"Auto-update check failed on startup: {e}") heartbeat_thread = threading.Thread( target=run_heartbeat, args=(args.panel_url, args.token), daemon=True, ) heartbeat_thread.start() logger.info(f"Heartbeat enabled → {args.panel_url}") # Start self-healing background monitor (port fixes, crash recovery) healing_thread = threading.Thread( target=_background_self_healing_loop, args=(300,), # every 5 minutes kwargs={"panel_url": args.panel_url, "token": args.token}, daemon=True, ) healing_thread.start() logger.info("Self-healing monitor enabled (checks every 5 minutes)") # Start dedicated fraud scanner (every 60 seconds) fraud_thread = threading.Thread( target=_background_fraud_scanner, args=(60,), # scan every 60 seconds kwargs={"panel_url": args.panel_url, "token": args.token}, daemon=True, ) fraud_thread.start() logger.info("Fraud scanner enabled (scans every 60 seconds)") # Start HTTP server server = ThreadingHTTPServer((args.host, args.port), AgentRequestHandler) # SSL setup if not args.no_ssl: cert = args.ssl_cert or "/etc/game-agent/ssl/node.crt" key = args.ssl_key or "/etc/game-agent/ssl/node.key" # Fallback to legacy paths if not os.path.exists(cert): cert = args.ssl_cert or "/etc/game-agent/cert.pem" if not os.path.exists(key): key = args.ssl_key or "/etc/game-agent/key.pem" if os.path.exists(cert) and os.path.exists(key): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.minimum_version = ssl.TLSVersion.TLSv1_2 ctx.load_cert_chain(cert, key) server.socket = ctx.wrap_socket(server.socket, server_side=True) logger.info(f"SSL enabled: cert={cert}") else: logger.warning(f"SSL cert/key not found ({cert}, {key}). Generating self-signed certificate...") run_cmd( f'mkdir -p /etc/game-agent && ' f'openssl req -x509 -newkey rsa:2048 -keyout {key} -out {cert} ' f'-days 3650 -nodes -subj "/CN=game-agent"', timeout=30, ) if os.path.exists(cert) and os.path.exists(key): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.minimum_version = ssl.TLSVersion.TLSv1_2 ctx.load_cert_chain(cert, key) server.socket = ctx.wrap_socket(server.socket, server_side=True) logger.info("SSL enabled with self-signed certificate") else: logger.warning("Failed to generate SSL certificate. Running without SSL.") # Graceful shutdown def signal_handler(sig, frame): logger.info("Shutting down Game Agent...") # shutdown() must be called from a different thread to avoid deadlock threading.Thread(target=server.shutdown, daemon=True).start() signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) proto = "https" if not args.no_ssl else "http" logger.info(f"IkaByte Game Agent v{AGENT_VERSION} listening on {proto}://{args.host}:{args.port}") logger.info(f"Data directory: {DATA_DIR}") try: server.serve_forever() except KeyboardInterrupt: pass finally: server.server_close() logger.info("Agent stopped.") if __name__ == "__main__": main()