diff --git a/CLAUDE.md b/CLAUDE.md index b9f0a2f..a2bfcfe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ bash scripts/deploy_model.sh --symbol XRPUSDT 4. `src/exchange.py` + `src/risk_manager.py` — Dynamic margin, MARKET orders with SL/TP, daily loss limit (5%), same-direction limit 5. `src/user_data_stream.py` + `src/notifier.py` — Real-time TP/SL detection via WebSocket, Discord webhooks -**Dual-layer kill switch** (per-symbol, in `src/bot.py`): Fast Kill (8 consecutive net losses) + Slow Kill (last 15 trades PF < 0.75). Trade history persisted to `data/trade_history/{symbol}.jsonl`. Blocks new entries only; existing SL/TP exits work normally. Manual reset via `RESET_KILL_SWITCH_{SYMBOL}=True` env var + restart. +**Dual-layer kill switch** (per-symbol, in `src/bot.py` and `src/mtf_bot.py`): Fast Kill (8 consecutive net losses) + Slow Kill (last 15 trades PF < 0.75). Trade history persisted to `data/trade_history/{symbol}.jsonl`. Blocks new entries only; existing SL/TP exits work normally. Manual reset via `RESET_KILL_SWITCH_{SYMBOL}=True` (main bot) or `RESET_KILL_SWITCH_MTF_{SYMBOL}=True` (MTF bot) env var + restart. MTF bot uses bps-based PnL for kill switch decisions. **Parallel execution**: Per-symbol bots run independently via `asyncio.gather()`. Each bot's `user_data_stream` also runs in parallel. @@ -155,3 +155,5 @@ All design documents and implementation plans are stored in `docs/plans/` with t | 2026-03-30 | `fr-oi-backtest` (result) | SHORT PF=1.88이나 대칭성 실패(Case2), 폐기 | | 2026-03-30 | `public-api-research-closed` | Binance 공개 API 전수 테스트 완료, 단독 edge 없음 | | 2026-03-30 | `mtf-pullback-bot` | MTF Pullback Bot 배포, 4월 OOS Dry-run 검증 진행 중 | +| 2026-04-21 | `mtf-oos-dryrun-result` | 중간 보고 — 24건 Raw PF 0.98 | +| 2026-05-04 | `mtf-oos-final-result` | **FAIL, 폐기** — 30건 fees_only PF 0.84, SHORT 대칭성 실패 | diff --git a/docs/plans/2026-05-04-mtf-oos-final-result.md b/docs/plans/2026-05-04-mtf-oos-final-result.md new file mode 100644 index 0000000..16831db --- /dev/null +++ b/docs/plans/2026-05-04-mtf-oos-final-result.md @@ -0,0 +1,48 @@ +# MTF Pullback Bot OOS Dry-run 최종 결과 + +## 가설 +MTF Pullback Bot의 멀티타임프레임 풀백 시그널이 XRPUSDT LONG/SHORT 양방향에서 수익을 낸다. + +## 데이터 +- 심볼: XRPUSDT +- 기간: 2026-04-02 ~ 2026-04-29 (28일) +- 거래 수: 30건 (LONG 18, SHORT 12) +- 실제 자금 투입: 0 (dry-run, 시그널+주문 기록만) + +## 결과 + +### Raw (비용 미반영, 참고용) + +| 방향 | 거래 수 | 승률 | PF | CumPnL(bps) | 평균보유 | +|------|---------|------|----|-------------|---------| +| Total | 30 | 43.3% | 1.06 | +61.6 | 237m | +| LONG | 18 | 50.0% | 1.28 | +167.7 | 272m | +| SHORT | 12 | 33.3% | 0.73 | -106.1 | 185m | + +### 비용 보정 시나리오 + +| 시나리오 | Total PF | Total CumPnL | LONG PF | SHORT PF | +|----------|----------|-------------|---------|----------| +| fees_only | 0.84 | -178.4 | 1.04 | 0.56 | +| realistic | 0.79 | -250.4 | 0.98 | 0.52 | +| pessimistic | 0.69 | -382.4 | 0.87 | 0.45 | + +### 4/21 중간 보고 대비 변화 + +| 지표 | 중간(24건) | 최종(30건) | +|------|-----------|-----------| +| Raw Total PF | 0.98 | 1.06 | +| fees_only Total PF | 0.79 | 0.84 | +| SHORT Raw PF | 0.58 | 0.73 | +| SHORT fees_only PF | 0.46 | 0.56 | + +마지막 6건에서 소폭 개선됐으나 구조적 문제(SHORT 역 edge, 비용 흡수 불가) 불변. + +## 결론 + +**FAIL** — 폐기 사유 2개 해당: + +1. **OOS PF < 1.0 (비용 반영)**: fees_only 시나리오에서도 PF 0.84, 수수료를 이길 수 없음 +2. **LONG/SHORT 대칭성 실패**: SHORT Raw PF 0.73 — 전략 자체에 SHORT edge 없음 + +Raw PF 1.06은 수수료(taker 왕복 8bps)를 흡수하기엔 마진이 너무 얇음. LONG만 분리 운영해도 fees_only PF 1.04로 거래 비용 감당 불가. diff --git a/scripts/evaluate_oos.py b/scripts/evaluate_oos.py index 82cafb5..e4dd58e 100644 --- a/scripts/evaluate_oos.py +++ b/scripts/evaluate_oos.py @@ -4,10 +4,15 @@ MTF Pullback Bot — OOS Dry-run 평가 스크립트 프로덕션 서버에서 JSONL 거래 기록을 가져와 승률·PF·누적PnL·평균보유시간을 계산하고 LIVE 배포 판정을 출력한다. +비용 모델(수수료·슬리피지·펀딩)을 사후보정으로 적용하여 +fees_only / realistic / pessimistic 3개 시나리오 결과를 출력한다. + Usage: python scripts/evaluate_oos.py python scripts/evaluate_oos.py --symbol xrpusdt python scripts/evaluate_oos.py --local # 로컬 파일만 사용 (서버 fetch 스킵) + python scripts/evaluate_oos.py --local --scenario all + python scripts/evaluate_oos.py --local --scenario fees_only """ import argparse @@ -17,6 +22,10 @@ from pathlib import Path import pandas as pd +# ── 비용 모델 import ───────────────────────────────────────────── +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from src.config import COST_MODEL, COST_SCENARIOS # noqa: E402 + # ── 설정 ────────────────────────────────────────────────────────── PROD_HOST = "root@10.1.10.24" REMOTE_DIR = "/root/cointrader/data/trade_history" @@ -67,20 +76,74 @@ def load_trades(path: Path) -> pd.DataFrame: return df -def calc_metrics(df: pd.DataFrame) -> dict: +def count_funding_events(entry_ts, exit_ts) -> int: + """ + Binance USDⓈ-M Futures 펀딩 스냅샷 시각(00/08/16 UTC)이 + [entry_ts, exit_ts] 구간에 몇 번 포함되는지 카운트. + """ + start = entry_ts.ceil("h") + end = exit_ts.floor("h") + if start > end: + return 0 + hours = pd.date_range(start, end, freq="1h", inclusive="both") + return sum(1 for h in hours if h.hour % 8 == 0) + + +def _get_fee_bps(order_type: str) -> float: + """주문 타입에 따른 수수료 bps 반환.""" + if order_type == "taker": + return COST_MODEL["taker_fee_bps"] + return COST_MODEL["maker_fee_bps"] + + +def calc_trade_cost(row, scenario: dict) -> float: + """개별 거래의 총 비용(bps)을 계산.""" + # 1) Fee: entry + exit + entry_fee = _get_fee_bps(COST_MODEL["entry_order_type"]) + + # exit order type: SL 히트면 sl_order_type, TP 히트면 tp_order_type + reason = row.get("reason", "") + if "SL" in reason: + exit_fee = _get_fee_bps(COST_MODEL["sl_order_type"]) + else: + exit_fee = _get_fee_bps(COST_MODEL["tp_order_type"]) + + fee = entry_fee + exit_fee + + # 2) Slippage: 왕복 + slippage = scenario["slippage_bps_per_side"] * 2 + + # 3) Funding: 경계 교차 카운트 + funding_count = count_funding_events(row["entry_ts"], row["exit_ts"]) + funding = funding_count * scenario["funding_bps_per_8h"] + + return fee + slippage + funding + + +def apply_cost_model(df: pd.DataFrame, scenario_name: str) -> pd.DataFrame: + """DataFrame에 비용을 적용하여 adjusted_pnl_bps 컬럼 추가.""" + scenario = COST_SCENARIOS[scenario_name] + result = df.copy() + result["cost_bps"] = result.apply(lambda row: calc_trade_cost(row, scenario), axis=1) + result["adjusted_pnl_bps"] = result["pnl_bps"] - result["cost_bps"] + return result + + +def calc_metrics(df: pd.DataFrame, pnl_col: str = "pnl_bps") -> dict: """핵심 지표 계산. 빈 DataFrame이면 안전한 기본값 반환.""" n = len(df) if n == 0: - return {"trades": 0, "win_rate": 0.0, "pf": 0.0, "cum_pnl": 0.0, "avg_dur": 0.0} + return {"trades": 0, "win_rate": 0.0, "pf": 0.0, "cum_pnl": 0.0, "avg_pnl": 0.0, "avg_dur": 0.0} - wins = df[df["pnl_bps"] > 0] - losses = df[df["pnl_bps"] < 0] + wins = df[df[pnl_col] > 0] + losses = df[df[pnl_col] < 0] win_rate = len(wins) / n * 100 - gross_profit = wins["pnl_bps"].sum() if len(wins) > 0 else 0.0 - gross_loss = abs(losses["pnl_bps"].sum()) if len(losses) > 0 else 0.0 + gross_profit = wins[pnl_col].sum() if len(wins) > 0 else 0.0 + gross_loss = abs(losses[pnl_col].sum()) if len(losses) > 0 else 0.0 pf = gross_profit / gross_loss if gross_loss > 0 else float("inf") - cum_pnl = df["pnl_bps"].sum() + cum_pnl = df[pnl_col].sum() + avg_pnl = cum_pnl / n avg_dur = df["duration_min"].mean() return { @@ -88,28 +151,29 @@ def calc_metrics(df: pd.DataFrame) -> dict: "win_rate": round(win_rate, 1), "pf": round(pf, 2), "cum_pnl": round(cum_pnl, 1), + "avg_pnl": round(avg_pnl, 2), "avg_dur": round(avg_dur, 1), } def print_report(df: pd.DataFrame): - """성적표 출력.""" + """성적표 출력 (raw, 비용 미반영).""" total = calc_metrics(df) longs = calc_metrics(df[df["side"] == "LONG"]) shorts = calc_metrics(df[df["side"] == "SHORT"]) header = f"{'':>10} {'Trades':>8} {'WinRate':>9} {'PF':>8} {'CumPnL':>10} {'AvgDur':>10}" - sep = "─" * 60 + sep = "\u2500" * 60 print() print(sep) - print(" MTF Pullback Bot — OOS Dry-run 성적표") + print(" MTF Pullback Bot \u2014 OOS Dry-run \uc131\uc801\ud45c") print(sep) print(header) print(sep) for label, m in [("Total", total), ("LONG", longs), ("SHORT", shorts)]: - pf_str = f"{m['pf']:.2f}" if m["pf"] != float("inf") else "∞" + pf_str = f"{m['pf']:.2f}" if m["pf"] != float("inf") else "\u221e" dur_str = f"{m['avg_dur']:.0f}m" if m["trades"] > 0 else "-" print( f"{label:>10} {m['trades']:>8d} {m['win_rate']:>8.1f}% {pf_str:>8} " @@ -128,46 +192,168 @@ def print_report(df: pd.DataFrame): dur = f"{row['duration_min']:.0f}m" reason = row.get("reason", "") if len(reason) > 25: - reason = reason[:25] + "…" + reason = reason[:25] + "\u2026" print( f"{i+1:>3} {row['side']:>6} {row['entry_price']:>10.4f} {row['exit_price']:>10.4f} " f"{row['pnl_bps']:>+10.1f} {dur:>8} {reason}" ) print(sep) - # ── 최종 판정 ── + # ── 최종 판정 (비용 반영 기준) ── + # Raw PF는 비현실적 — fees_only 기준으로 판정 + cost_df = apply_cost_model(df, "fees_only") + cost_total = calc_metrics(cost_df, pnl_col="adjusted_pnl_bps") + cost_long = calc_metrics(cost_df[cost_df["side"] == "LONG"], pnl_col="adjusted_pnl_bps") + cost_short = calc_metrics(cost_df[cost_df["side"] == "SHORT"], pnl_col="adjusted_pnl_bps") + + # 대칭성 체크: LONG/SHORT 양쪽 모두 PF >= 0.8 이상이어야 함 + symmetry_ok = True + if cost_long["trades"] >= 5 and cost_short["trades"] >= 5: + symmetry_ok = cost_long["pf"] >= 0.8 and cost_short["pf"] >= 0.8 + print() - if total["trades"] >= MIN_TRADES and total["pf"] >= MIN_PF: - print(f" [판정: 통과] 엣지가 증명되었습니다. LIVE 배포(자금 투입)를 권장합니다.") - print(f" (거래수 {total['trades']} >= {MIN_TRADES}, PF {total['pf']:.2f} >= {MIN_PF:.1f})") + if cost_total["trades"] >= MIN_TRADES and cost_total["pf"] >= MIN_PF and symmetry_ok: + print(f" [\ud310\uc815: \ud1b5\uacfc] \uc5e3\uc9c0\uac00 \uc99d\uba85\ub418\uc5c8\uc2b5\ub2c8\ub2e4. LIVE \ubc30\ud3ec(\uc790\uae08 \ud22c\uc785)\ub97c \uad8c\uc7a5\ud569\ub2c8\ub2e4.") + print(f" (\uac70\ub798\uc218 {cost_total['trades']} >= {MIN_TRADES}, fees_only PF {cost_total['pf']:.2f} >= {MIN_PF:.1f})") else: reasons = [] - if total["trades"] < MIN_TRADES: - reasons.append(f"거래수 {total['trades']} < {MIN_TRADES}") - if total["pf"] < MIN_PF: - reasons.append(f"PF {total['pf']:.2f} < {MIN_PF:.1f}") - print(f" [판정: 보류] 기준 미달. OOS 검증 실패로 실전 투입을 보류합니다.") + if cost_total["trades"] < MIN_TRADES: + reasons.append(f"\uac70\ub798\uc218 {cost_total['trades']} < {MIN_TRADES}") + if cost_total["pf"] < MIN_PF: + reasons.append(f"fees_only PF {cost_total['pf']:.2f} < {MIN_PF:.1f}") + if not symmetry_ok: + reasons.append(f"LONG/SHORT \ube44\ub300\uce6d (L:{cost_long['pf']:.2f} / S:{cost_short['pf']:.2f})") + print(f" [\ud310\uc815: \uc2e4\ud328] OOS \uac80\uc99d \uc2e4\ud328. \uc2e4\uc804 \ud22c\uc785 \ubd88\uac00.") print(f" ({', '.join(reasons)})") print() +def print_cost_report(df: pd.DataFrame, scenario_names: list[str]): + """비용 보정 시나리오별 성적표 출력.""" + sep = "\u2500" * 61 + + # 시나리오별 데이터 준비 + scenario_dfs = {} + for name in scenario_names: + scenario_dfs[name] = apply_cost_model(df, name) + + print() + print(sep) + print(" MTF Pullback Bot \u2014 OOS Cost-Adjusted Results") + print(sep) + + # 헤더 + header = f"{'Scenario:':>16}" + for name in scenario_names: + header += f" {name:>14}" + print(header) + print(sep) + + # Total / LONG / SHORT 각각 + for section_label, filter_fn in [ + ("Total", lambda d: d), + ("LONG", lambda d: d[d["side"] == "LONG"]), + ("SHORT", lambda d: d[d["side"] == "SHORT"]), + ]: + print(section_label) + + # 각 시나리오에 대해 metrics 계산 + metrics_list = [] + for name in scenario_names: + sdf = filter_fn(scenario_dfs[name]) + m = calc_metrics(sdf, pnl_col="adjusted_pnl_bps") + metrics_list.append(m) + + # Trades + line = f"{'Trades:':>16}" + for m in metrics_list: + line += f" {m['trades']:>14d}" + print(line) + + # WinRate + line = f"{'WinRate:':>16}" + for m in metrics_list: + line += f" {m['win_rate']:>13.1f}%" + print(line) + + # PF + line = f"{'PF:':>16}" + for m in metrics_list: + pf_str = f"{m['pf']:.2f}" if m["pf"] != float("inf") else "\u221e" + line += f" {pf_str:>14}" + print(line) + + # CumPnL(bps) + line = f"{'CumPnL(bps):':>16}" + for m in metrics_list: + line += f" {m['cum_pnl']:>+14.1f}" + print(line) + + # AvgPnL(bps) + line = f"{'AvgPnL(bps):':>16}" + for m in metrics_list: + line += f" {m['avg_pnl']:>+14.2f}" + print(line) + + # AvgDur + line = f"{'AvgDur:':>16}" + for m in metrics_list: + dur_str = f"{m['avg_dur']:.0f}m" if m["trades"] > 0 else "-" + line += f" {dur_str:>14}" + print(line) + + print(sep) + + # Raw 참고 + raw_total = calc_metrics(df) + print(f"Raw (\ube44\uc6a9 \ubbf8\ubc18\uc601, \ucc38\uace0\uc6a9):") + pf_str = f"{raw_total['pf']:.2f}" if raw_total["pf"] != float("inf") else "\u221e" + print(f" Total PF: {pf_str}, CumPnL: {raw_total['cum_pnl']:+.1f} bps") + print(sep) + print() + + def main(): - parser = argparse.ArgumentParser(description="MTF OOS Dry-run 평가") - parser.add_argument("--symbol", default="xrpusdt", help="심볼 (파일명 소문자, 기본: xrpusdt)") - parser.add_argument("--local", action="store_true", help="로컬 파일만 사용 (서버 fetch 스킵)") + parser = argparse.ArgumentParser(description="MTF OOS Dry-run \ud3c9\uac00") + parser.add_argument("--symbol", default="xrpusdt", help="\uc2ec\ubcfc (\ud30c\uc77c\uba85 \uc18c\ubb38\uc790, \uae30\ubcf8: xrpusdt)") + parser.add_argument("--local", action="store_true", help="\ub85c\uceec \ud30c\uc77c\ub9cc \uc0ac\uc6a9 (\uc11c\ubc84 fetch \uc2a4\ud0b5)") + parser.add_argument( + "--scenario", + choices=["fees_only", "realistic", "pessimistic", "all"], + default="all", + help="\ube44\uc6a9 \ubcf4\uc815 \uc2dc\ub098\ub9ac\uc624 (\uae30\ubcf8: all)", + ) args = parser.parse_args() - filename = f"mtf_{args.symbol}.jsonl" + # MTF bot은 ccxt 심볼(XRP/USDT:USDT)에서 /,:를 제거하여 파일명 생성 + # → mtf_xrpusdtusdt.jsonl (심볼 인자 xrpusdt → xrpusdtusdt 변환) + raw = args.symbol.lower() + if not raw.endswith("usdt"): + raw = raw + "usdt" + # xrpusdt → xrpusdtusdt (ccxt 포맷 XRP/USDT:USDT 의 슬래시·콜론 제거 결과) + if raw.endswith("usdt") and not raw.endswith("usdtusdt"): + raw = raw + "usdt" + filename = f"mtf_{raw}.jsonl" if args.local: local_path = LOCAL_DIR / filename if not local_path.exists(): - print(f"[Error] 로컬 파일 없음: {local_path}") + print(f"[Error] \ub85c\uceec \ud30c\uc77c \uc5c6\uc74c: {local_path}") sys.exit(1) else: local_path = fetch_from_prod(filename) df = load_trades(local_path) + + # 비용 보정 리포트 출력 + if args.scenario == "all": + scenario_names = ["fees_only", "realistic", "pessimistic"] + else: + scenario_names = [args.scenario] + + print_cost_report(df, scenario_names) + + # raw 리포트도 하단에 유지 print_report(df)