565414c5e0
- Introduced a new trading dashboard consisting of a FastAPI backend (`dashboard-api`) for data retrieval and a React frontend (`dashboard-ui`) for visualization. - Implemented a log parser to monitor and store bot logs in an SQLite database. - Configured Docker setup for both API and UI, including necessary Dockerfiles and a docker-compose configuration. - Added setup documentation for running the dashboard and accessing its features. - Enhanced the Jenkins pipeline to build and push the new dashboard images.
477 lines
18 KiB
Python
477 lines
18 KiB
Python
"""
|
|
log_parser.py — 봇 로그 파일을 감시하고 파싱하여 SQLite에 저장
|
|
봇 코드 수정 없이 동작. logs/ 디렉토리만 마운트하면 됨.
|
|
|
|
실행: python log_parser.py
|
|
"""
|
|
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
import glob
|
|
import os
|
|
import json
|
|
import threading
|
|
from datetime import datetime, date
|
|
from pathlib import Path
|
|
|
|
# ── 설정 ──────────────────────────────────────────────────────────
|
|
LOG_DIR = os.environ.get("LOG_DIR", "/app/logs")
|
|
DB_PATH = os.environ.get("DB_PATH", "/app/data/dashboard.db")
|
|
POLL_INTERVAL = int(os.environ.get("POLL_INTERVAL", "5")) # 초
|
|
|
|
# ── 정규식 패턴 (실제 봇 로그 형식 기준) ──────────────────────────
|
|
PATTERNS = {
|
|
# 신호: HOLD | 현재가: 1.3889 USDT
|
|
"signal": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*신호: (?P<signal>\w+) \| 현재가: (?P<price>[\d.]+) USDT"
|
|
),
|
|
|
|
# ADX: 24.4
|
|
"adx": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*ADX: (?P<adx>[\d.]+)"
|
|
),
|
|
|
|
# OI=261103765.6, OI변화율=0.000692, 펀딩비=0.000039
|
|
"microstructure": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*OI=(?P<oi>[\d.]+), OI변화율=(?P<oi_change>[-\d.]+), 펀딩비=(?P<funding>[-\d.]+)"
|
|
),
|
|
|
|
# 기존 포지션 복구: SHORT | 진입가=1.4126 | 수량=86.8
|
|
"position_recover": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*기존 포지션 복구: (?P<direction>\w+) \| 진입가=(?P<entry_price>[\d.]+) \| 수량=(?P<qty>[\d.]+)"
|
|
),
|
|
|
|
# SHORT 진입: 가격=1.3940, 수량=86.8, SL=1.4040, TP=1.3840
|
|
"entry": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*(?P<direction>SHORT|LONG) 진입: "
|
|
r"가격=(?P<entry_price>[\d.]+), "
|
|
r"수량=(?P<qty>[\d.]+), "
|
|
r"SL=(?P<sl>[\d.]+), "
|
|
r"TP=(?P<tp>[\d.]+)"
|
|
),
|
|
|
|
# 청산 감지(MANUAL): exit=1.3782, rp=+2.9859, commission=0.0598, net_pnl=+2.9261
|
|
"close_detect": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*청산 감지\((?P<reason>\w+)\):\s*"
|
|
r"exit=(?P<exit_price>[\d.]+),\s*"
|
|
r"rp=(?P<expected>[+\-\d.]+),\s*"
|
|
r"commission=(?P<commission>[\d.]+),\s*"
|
|
r"net_pnl=(?P<net_pnl>[+\-\d.]+)"
|
|
),
|
|
|
|
# 오늘 누적 PnL: 2.9261 USDT
|
|
"daily_pnl": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*오늘 누적 PnL: (?P<pnl>[+\-\d.]+) USDT"
|
|
),
|
|
|
|
# 봇 시작: XRPUSDT, 레버리지 10x
|
|
"bot_start": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*봇 시작: (?P<symbol>\w+), 레버리지 (?P<leverage>\d+)x"
|
|
),
|
|
|
|
# 기준 잔고 설정: 24.46 USDT
|
|
"balance": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*기준 잔고 설정: (?P<balance>[\d.]+) USDT"
|
|
),
|
|
|
|
# ML 필터 로드
|
|
"ml_filter": re.compile(
|
|
r"(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
|
|
r".*ML 필터 로드.*임계값=(?P<threshold>[\d.]+)"
|
|
),
|
|
}
|
|
|
|
|
|
class LogParser:
|
|
def __init__(self):
|
|
Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True)
|
|
self.conn = sqlite3.connect(DB_PATH)
|
|
self.conn.row_factory = sqlite3.Row
|
|
self.conn.execute("PRAGMA journal_mode=WAL")
|
|
self._init_db()
|
|
|
|
# 상태 추적
|
|
self._file_positions = {} # {파일경로: 마지막 읽은 위치}
|
|
self._current_position = None # 현재 열린 포지션 정보
|
|
self._pending_candle = {} # 타임스탬프 기준으로 지표를 모아두기
|
|
self._bot_config = {"symbol": "XRPUSDT", "leverage": 10}
|
|
self._balance = 0
|
|
|
|
def _init_db(self):
|
|
self.conn.executescript("""
|
|
CREATE TABLE IF NOT EXISTS trades (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
symbol TEXT NOT NULL DEFAULT 'XRPUSDT',
|
|
direction TEXT NOT NULL,
|
|
entry_time TEXT NOT NULL,
|
|
exit_time TEXT,
|
|
entry_price REAL NOT NULL,
|
|
exit_price REAL,
|
|
quantity REAL,
|
|
leverage INTEGER DEFAULT 10,
|
|
sl REAL,
|
|
tp REAL,
|
|
rsi REAL,
|
|
macd_hist REAL,
|
|
atr REAL,
|
|
adx REAL,
|
|
expected_pnl REAL,
|
|
actual_pnl REAL,
|
|
commission REAL,
|
|
net_pnl REAL,
|
|
status TEXT NOT NULL DEFAULT 'OPEN',
|
|
close_reason TEXT,
|
|
extra TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS candles (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
ts TEXT NOT NULL UNIQUE,
|
|
price REAL NOT NULL,
|
|
signal TEXT,
|
|
adx REAL,
|
|
oi REAL,
|
|
oi_change REAL,
|
|
funding_rate REAL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS daily_pnl (
|
|
date TEXT PRIMARY KEY,
|
|
cumulative_pnl REAL DEFAULT 0,
|
|
trade_count INTEGER DEFAULT 0,
|
|
wins INTEGER DEFAULT 0,
|
|
losses INTEGER DEFAULT 0,
|
|
last_updated TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS bot_status (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT,
|
|
updated_at TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS parse_state (
|
|
filepath TEXT PRIMARY KEY,
|
|
position INTEGER DEFAULT 0
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_candles_ts ON candles(ts);
|
|
CREATE INDEX IF NOT EXISTS idx_trades_status ON trades(status);
|
|
""")
|
|
self.conn.commit()
|
|
self._load_state()
|
|
|
|
def _load_state(self):
|
|
"""이전 파싱 위치 복원"""
|
|
rows = self.conn.execute("SELECT filepath, position FROM parse_state").fetchall()
|
|
self._file_positions = {r["filepath"]: r["position"] for r in rows}
|
|
|
|
# 현재 열린 포지션 복원
|
|
row = self.conn.execute(
|
|
"SELECT * FROM trades WHERE status='OPEN' ORDER BY id DESC LIMIT 1"
|
|
).fetchone()
|
|
if row:
|
|
self._current_position = dict(row)
|
|
|
|
def _save_position(self, filepath, pos):
|
|
self.conn.execute(
|
|
"INSERT INTO parse_state(filepath, position) VALUES(?,?) "
|
|
"ON CONFLICT(filepath) DO UPDATE SET position=?",
|
|
(filepath, pos, pos)
|
|
)
|
|
self.conn.commit()
|
|
|
|
def _set_status(self, key, value):
|
|
now = datetime.now().isoformat()
|
|
self.conn.execute(
|
|
"INSERT INTO bot_status(key, value, updated_at) VALUES(?,?,?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value=?, updated_at=?",
|
|
(key, str(value), now, str(value), now)
|
|
)
|
|
self.conn.commit()
|
|
|
|
# ── 메인 루프 ────────────────────────────────────────────────
|
|
def run(self):
|
|
print(f"[LogParser] 시작 — LOG_DIR={LOG_DIR}, DB={DB_PATH}, 폴링={POLL_INTERVAL}s")
|
|
while True:
|
|
try:
|
|
self._scan_logs()
|
|
except Exception as e:
|
|
print(f"[LogParser] 에러: {e}")
|
|
time.sleep(POLL_INTERVAL)
|
|
|
|
def _scan_logs(self):
|
|
"""로그 파일 목록을 가져와서 새 줄 파싱"""
|
|
# 날짜 형식 (bot_2026-03-01.log) + 현재 형식 (bot.log) 모두 스캔
|
|
log_files = sorted(glob.glob(os.path.join(LOG_DIR, "bot_*.log")))
|
|
main_log = os.path.join(LOG_DIR, "bot.log")
|
|
if os.path.exists(main_log):
|
|
log_files.append(main_log)
|
|
for filepath in log_files:
|
|
self._parse_file(filepath)
|
|
|
|
def _parse_file(self, filepath):
|
|
last_pos = self._file_positions.get(filepath, 0)
|
|
|
|
try:
|
|
file_size = os.path.getsize(filepath)
|
|
except OSError:
|
|
return
|
|
|
|
# 파일이 줄었으면 (로테이션) 처음부터
|
|
if file_size < last_pos:
|
|
last_pos = 0
|
|
|
|
if file_size == last_pos:
|
|
return # 새 내용 없음
|
|
|
|
with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
|
|
f.seek(last_pos)
|
|
new_lines = f.readlines()
|
|
new_pos = f.tell()
|
|
|
|
for line in new_lines:
|
|
self._parse_line(line.strip())
|
|
|
|
self._file_positions[filepath] = new_pos
|
|
self._save_position(filepath, new_pos)
|
|
|
|
# ── 한 줄 파싱 ──────────────────────────────────────────────
|
|
def _parse_line(self, line):
|
|
if not line:
|
|
return
|
|
|
|
# 봇 시작
|
|
m = PATTERNS["bot_start"].search(line)
|
|
if m:
|
|
self._bot_config["symbol"] = m.group("symbol")
|
|
self._bot_config["leverage"] = int(m.group("leverage"))
|
|
self._set_status("symbol", m.group("symbol"))
|
|
self._set_status("leverage", m.group("leverage"))
|
|
self._set_status("last_start", m.group("ts"))
|
|
return
|
|
|
|
# 잔고
|
|
m = PATTERNS["balance"].search(line)
|
|
if m:
|
|
self._balance = float(m.group("balance"))
|
|
self._set_status("balance", m.group("balance"))
|
|
return
|
|
|
|
# ML 필터
|
|
m = PATTERNS["ml_filter"].search(line)
|
|
if m:
|
|
self._set_status("ml_threshold", m.group("threshold"))
|
|
return
|
|
|
|
# 포지션 복구 (재시작 시)
|
|
m = PATTERNS["position_recover"].search(line)
|
|
if m:
|
|
self._handle_entry(
|
|
ts=m.group("ts"),
|
|
direction=m.group("direction"),
|
|
entry_price=float(m.group("entry_price")),
|
|
qty=float(m.group("qty")),
|
|
is_recovery=True,
|
|
)
|
|
return
|
|
|
|
# 포지션 진입: SHORT 진입: 가격=X, 수량=Y, SL=Z, TP=W
|
|
m = PATTERNS["entry"].search(line)
|
|
if m:
|
|
self._handle_entry(
|
|
ts=m.group("ts"),
|
|
direction=m.group("direction"),
|
|
entry_price=float(m.group("entry_price")),
|
|
qty=float(m.group("qty")),
|
|
sl=float(m.group("sl")),
|
|
tp=float(m.group("tp")),
|
|
)
|
|
return
|
|
|
|
# OI/펀딩비 (캔들 데이터에 합침)
|
|
m = PATTERNS["microstructure"].search(line)
|
|
if m:
|
|
ts_key = m.group("ts")[:16] # 분 단위로 그룹
|
|
if ts_key not in self._pending_candle:
|
|
self._pending_candle[ts_key] = {}
|
|
self._pending_candle[ts_key].update({
|
|
"oi": float(m.group("oi")),
|
|
"oi_change": float(m.group("oi_change")),
|
|
"funding": float(m.group("funding")),
|
|
})
|
|
return
|
|
|
|
# ADX
|
|
m = PATTERNS["adx"].search(line)
|
|
if m:
|
|
ts_key = m.group("ts")[:16]
|
|
if ts_key not in self._pending_candle:
|
|
self._pending_candle[ts_key] = {}
|
|
self._pending_candle[ts_key]["adx"] = float(m.group("adx"))
|
|
return
|
|
|
|
# 신호 + 현재가 → 캔들 저장
|
|
m = PATTERNS["signal"].search(line)
|
|
if m:
|
|
ts = m.group("ts")
|
|
ts_key = ts[:16]
|
|
price = float(m.group("price"))
|
|
signal = m.group("signal")
|
|
extra = self._pending_candle.pop(ts_key, {})
|
|
|
|
self._set_status("current_price", str(price))
|
|
self._set_status("current_signal", signal)
|
|
self._set_status("last_candle_time", ts)
|
|
|
|
try:
|
|
self.conn.execute(
|
|
"""INSERT INTO candles(ts, price, signal, adx, oi, oi_change, funding_rate)
|
|
VALUES(?,?,?,?,?,?,?)
|
|
ON CONFLICT(ts) DO UPDATE SET
|
|
price=?, signal=?, adx=?, oi=?, oi_change=?, funding_rate=?""",
|
|
(ts, price, signal,
|
|
extra.get("adx"), extra.get("oi"), extra.get("oi_change"), extra.get("funding"),
|
|
price, signal,
|
|
extra.get("adx"), extra.get("oi"), extra.get("oi_change"), extra.get("funding")),
|
|
)
|
|
self.conn.commit()
|
|
except Exception as e:
|
|
print(f"[LogParser] 캔들 저장 에러: {e}")
|
|
return
|
|
|
|
# 청산 감지
|
|
m = PATTERNS["close_detect"].search(line)
|
|
if m:
|
|
self._handle_close(
|
|
ts=m.group("ts"),
|
|
exit_price=float(m.group("exit_price")),
|
|
expected_pnl=float(m.group("expected")),
|
|
commission=float(m.group("commission")),
|
|
net_pnl=float(m.group("net_pnl")),
|
|
reason=m.group("reason"),
|
|
)
|
|
return
|
|
|
|
# 일일 누적 PnL
|
|
m = PATTERNS["daily_pnl"].search(line)
|
|
if m:
|
|
ts = m.group("ts")
|
|
day = ts[:10]
|
|
pnl = float(m.group("pnl"))
|
|
self.conn.execute(
|
|
"""INSERT INTO daily_pnl(date, cumulative_pnl, last_updated)
|
|
VALUES(?,?,?)
|
|
ON CONFLICT(date) DO UPDATE SET cumulative_pnl=?, last_updated=?""",
|
|
(day, pnl, ts, pnl, ts)
|
|
)
|
|
self.conn.commit()
|
|
self._set_status("daily_pnl", str(pnl))
|
|
return
|
|
|
|
# ── 포지션 진입 핸들러 ───────────────────────────────────────
|
|
def _handle_entry(self, ts, direction, entry_price, qty,
|
|
leverage=None, sl=None, tp=None, is_recovery=False):
|
|
if leverage is None:
|
|
leverage = self._bot_config.get("leverage", 10)
|
|
|
|
# 메모리 내 중복 체크
|
|
if self._current_position:
|
|
if abs(self._current_position["entry_price"] - entry_price) < 0.0001:
|
|
return
|
|
|
|
# DB 내 중복 체크 — 같은 방향·가격의 OPEN 포지션이 이미 있으면 스킵
|
|
existing = self.conn.execute(
|
|
"SELECT id FROM trades WHERE status='OPEN' AND direction=? "
|
|
"AND ABS(entry_price - ?) < 0.0001",
|
|
(direction, entry_price),
|
|
).fetchone()
|
|
if existing:
|
|
self._current_position = {
|
|
"id": existing["id"],
|
|
"direction": direction,
|
|
"entry_price": entry_price,
|
|
"entry_time": ts,
|
|
}
|
|
return
|
|
|
|
cur = self.conn.execute(
|
|
"""INSERT INTO trades(symbol, direction, entry_time, entry_price,
|
|
quantity, leverage, sl, tp, status, extra)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?)""",
|
|
(self._bot_config.get("symbol", "XRPUSDT"), direction, ts,
|
|
entry_price, qty, leverage, sl, tp, "OPEN",
|
|
json.dumps({"recovery": is_recovery})),
|
|
)
|
|
self.conn.commit()
|
|
self._current_position = {
|
|
"id": cur.lastrowid,
|
|
"direction": direction,
|
|
"entry_price": entry_price,
|
|
"entry_time": ts,
|
|
}
|
|
self._set_status("position_status", "OPEN")
|
|
self._set_status("position_direction", direction)
|
|
self._set_status("position_entry_price", str(entry_price))
|
|
print(f"[LogParser] 포지션 진입: {direction} @ {entry_price} (recovery={is_recovery})")
|
|
|
|
# ── 포지션 청산 핸들러 ───────────────────────────────────────
|
|
def _handle_close(self, ts, exit_price, expected_pnl, commission, net_pnl, reason):
|
|
if not self._current_position:
|
|
# 열린 포지션 없으면 DB에서 찾기
|
|
row = self.conn.execute(
|
|
"SELECT id, entry_price, direction FROM trades WHERE status='OPEN' ORDER BY id DESC LIMIT 1"
|
|
).fetchone()
|
|
if row:
|
|
self._current_position = dict(row)
|
|
else:
|
|
print(f"[LogParser] 경고: 청산 감지했으나 열린 포지션 없음")
|
|
return
|
|
|
|
trade_id = self._current_position["id"]
|
|
self.conn.execute(
|
|
"""UPDATE trades SET
|
|
exit_time=?, exit_price=?, expected_pnl=?,
|
|
actual_pnl=?, commission=?, net_pnl=?,
|
|
status='CLOSED', close_reason=?
|
|
WHERE id=?""",
|
|
(ts, exit_price, expected_pnl,
|
|
expected_pnl, commission, net_pnl,
|
|
reason, trade_id)
|
|
)
|
|
|
|
# 일별 요약 갱신
|
|
day = ts[:10]
|
|
win = 1 if net_pnl > 0 else 0
|
|
loss = 1 if net_pnl <= 0 else 0
|
|
self.conn.execute(
|
|
"""INSERT INTO daily_pnl(date, cumulative_pnl, trade_count, wins, losses, last_updated)
|
|
VALUES(?, ?, 1, ?, ?, ?)
|
|
ON CONFLICT(date) DO UPDATE SET
|
|
trade_count = trade_count + 1,
|
|
wins = wins + ?,
|
|
losses = losses + ?,
|
|
last_updated = ?""",
|
|
(day, net_pnl, win, loss, ts, win, loss, ts)
|
|
)
|
|
self.conn.commit()
|
|
|
|
self._set_status("position_status", "NONE")
|
|
print(f"[LogParser] 포지션 청산: {reason} @ {exit_price}, PnL={net_pnl}")
|
|
self._current_position = None
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = LogParser()
|
|
parser.run()
|