일일 통계 수집 기능을 추가하고, API 호출 및 네트워크 트래픽 기록을 개선하였습니다. 실시간 세션 및 워커 상태 조회 API를 구현하였으며, 대시보드에서 실시간 상태를 반영하도록 수정하였습니다. 상태 JSON 파일에 일일 통계 정보를 추가하였습니다.

This commit is contained in:
vast 2025-10-02 05:52:57 +00:00
parent 47ba96e148
commit bd7e4b4b33
16 changed files with 32303 additions and 89445 deletions

View File

@ -163,6 +163,34 @@ async def health_check_compat(request: Request):
return await health_check(request) return await health_check(request)
@router.get("/api/v1/realtime_status", include_in_schema=False)
async def get_realtime_status(request: Request):
"""실시간 세션/워커 상태 (모니터링 대시보드용)"""
try:
from app.core.session_pool import session_pool
from app.core.worker_manager import worker_manager
# 실시간 세션 상태 조회
session_status = session_pool.get_status()
# 실시간 워커 상태 조회
worker_status = worker_manager.get_status()
return {
"session_status": session_status,
"worker_status": worker_status,
"timestamp": time.time()
}
except Exception as e:
logger.error(f"실시간 상태 조회 실패: {e}")
return {
"session_status": {},
"worker_status": {},
"error": str(e),
"timestamp": time.time()
}
@router.get("/api/v1/server-config", response_model=ServerConfigResponse) @router.get("/api/v1/server-config", response_model=ServerConfigResponse)
async def get_server_config(): async def get_server_config():
"""서버 설정 정보 반환 (iopaint 호환)""" """서버 설정 정보 반환 (iopaint 호환)"""

View File

@ -90,14 +90,14 @@ class Settings(BaseSettings):
# 동적 세션 풀/메모리 # 동적 세션 풀/메모리
# ========================= # =========================
SIMPLE_LAMA_MIN_SESSIONS: int = 4 SIMPLE_LAMA_MIN_SESSIONS: int = 4
SIMPLE_LAMA_MAX_SESSIONS: int = 8 SIMPLE_LAMA_MAX_SESSIONS: int = 6
# x86에서는 MIGAN 미로딩(지연 로딩) 기본 → MIN=0 # x86에서는 MIGAN 미로딩(지연 로딩) 기본 → MIN=0
MIGAN_MIN_SESSIONS: int = 4 if IS_JETSON else 1 MIGAN_MIN_SESSIONS: int = 2 if IS_JETSON else 1
MIGAN_MAX_SESSIONS: int = 8 MIGAN_MAX_SESSIONS: int = 6
REMBG_MIN_SESSIONS: int = 3 if IS_JETSON else 1 REMBG_MIN_SESSIONS: int = 2
REMBG_MAX_SESSIONS: int = 6 if IS_JETSON else 4 REMBG_MAX_SESSIONS: int = 6
# 여유 VRAM 비율(남은 VRAM이 이 값보다 커야 세션 추가) # 여유 VRAM 비율(남은 VRAM이 이 값보다 커야 세션 추가)
SESSION_VRAM_THRESHOLD: float = 0.30 SESSION_VRAM_THRESHOLD: float = 0.30
@ -105,8 +105,8 @@ class Settings(BaseSettings):
# 마이크로 배치(SimpleLAMA) # 마이크로 배치(SimpleLAMA)
USE_MICRO_BATCHING: bool = True USE_MICRO_BATCHING: bool = True
MICRO_BATCH_SIZE: int = 8 MICRO_BATCH_SIZE: int = 4
MICRO_BATCH_TIMEOUT_MS: int = 80 MICRO_BATCH_TIMEOUT_MS: int = 100
# 사전 확정 세션(플랫폼 감안 기본치) # 사전 확정 세션(플랫폼 감안 기본치)
SIMPLE_LAMA_SESSIONS: int = 4 SIMPLE_LAMA_SESSIONS: int = 4
@ -115,7 +115,7 @@ class Settings(BaseSettings):
# 워커(내부 큐/스레드 워커, 프로세스는 WORKERS) # 워커(내부 큐/스레드 워커, 프로세스는 WORKERS)
MAX_WORKERS: int = 6 if IS_JETSON else 12 MAX_WORKERS: int = 6 if IS_JETSON else 12
MIN_WORKERS: int = 3 if IS_JETSON else 4 MIN_WORKERS: int = 2 if IS_JETSON else 6
WORKER_TIMEOUT: int = 120 WORKER_TIMEOUT: int = 120
# ========================= # =========================
@ -123,7 +123,7 @@ class Settings(BaseSettings):
# ========================= # =========================
VRAM_THRESHOLD_HIGH: float = 0.70 if IS_JETSON else 0.80 VRAM_THRESHOLD_HIGH: float = 0.70 if IS_JETSON else 0.80
VRAM_THRESHOLD_LOW: float = 0.30 if IS_JETSON else 0.40 VRAM_THRESHOLD_LOW: float = 0.30 if IS_JETSON else 0.40
VRAM_CHECK_INTERVAL: int = 10 if IS_JETSON else 5 # 초 VRAM_CHECK_INTERVAL: int = 20 if IS_JETSON else 15 # 초
# ========================= # =========================
# 모델/경로 # 모델/경로

View File

@ -14,7 +14,7 @@ from collections import defaultdict
from ..core.config import settings from ..core.config import settings
from ..utils.gpu_monitor import gpu_monitor from ..utils.gpu_monitor import gpu_monitor
from ..utils.monitor_events import append_event from ..utils.session_event_log import log_session_event
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -119,16 +119,7 @@ class SessionPool:
) )
logger.info(f"Successfully created session {session_id}") logger.info(f"Successfully created session {session_id}")
self._log_pool_status("create", model_type.value) self._log_pool_status("create", model_type.value)
try: log_session_event("session_create", model_type=model_type.value, session_id=session_id)
append_event({
"type": "session",
"action": "create",
"model": model_type.value,
"session_id": session_id,
"pool_size": len(self.pools[model_type]) + 1,
})
except Exception:
pass
return session return session
except Exception as e: except Exception as e:
logger.error(f"Failed to create session {session_id}: {e}", exc_info=True) logger.error(f"Failed to create session {session_id}: {e}", exc_info=True)
@ -262,17 +253,9 @@ class SessionPool:
for session in sessions_to_reap: for session in sessions_to_reap:
pool.remove(session) pool.remove(session)
reaped_counts[session.model_type.value] += 1 reaped_counts[session.model_type.value] += 1
log_session_event("session_destroy", model_type=session.model_type.value, session_id=session.session_id, details={"reason": "idle_timeout"})
del session.model del session.model
del session del session
try:
append_event({
"type": "session",
"action": "reap",
"model": model_type.value,
"pool_size": len(pool),
})
except Exception:
pass
self.conditions[model_type].notify_all() self.conditions[model_type].notify_all()

View File

@ -16,7 +16,6 @@ from ..utils.gpu_monitor import gpu_monitor
from ..core.config import settings from ..core.config import settings
from ..core.stats_manager import stats_manager from ..core.stats_manager import stats_manager
from ..core.session_pool import ModelType from ..core.session_pool import ModelType
from ..utils.monitor_events import append_event
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -231,32 +230,12 @@ class WorkerManager:
await self._scale_workers(new_count) await self._scale_workers(new_count)
self.last_scale_time = current_time self.last_scale_time = current_time
logger.info(f"Scaled up to {new_count} workers (VRAM: {vram_usage:.2f})") logger.info(f"Scaled up to {new_count} workers (VRAM: {vram_usage:.2f})")
try:
append_event({
"type": "worker_scale",
"action": "up",
"new_count": new_count,
"queue_size": queue_size,
"vram_usage": vram_usage,
})
except Exception:
pass
elif should_scale_down: elif should_scale_down:
new_count = max(total_workers - 1, settings.MIN_WORKERS) new_count = max(total_workers - 1, settings.MIN_WORKERS)
await self._scale_workers(new_count) await self._scale_workers(new_count)
self.last_scale_time = current_time self.last_scale_time = current_time
logger.info(f"Scaled down to {new_count} workers (VRAM: {vram_usage:.2f})") logger.info(f"Scaled down to {new_count} workers (VRAM: {vram_usage:.2f})")
try:
append_event({
"type": "worker_scale",
"action": "down",
"new_count": new_count,
"queue_size": queue_size,
"vram_usage": vram_usage,
})
except Exception:
pass
async def _scale_workers(self, target_count: int): async def _scale_workers(self, target_count: int):
"""워커 수를 조정합니다.""" """워커 수를 조정합니다."""

View File

@ -25,6 +25,8 @@ from ..core.worker_manager import worker_manager
from ..core.session_pool import session_pool from ..core.session_pool import session_pool
from ..utils.gpu_monitor import gpu_monitor from ..utils.gpu_monitor import gpu_monitor
from ..core.config import settings from ..core.config import settings
from ..utils.session_event_log import get_recent_events, read_events_from_file
from ..utils.daily_stats import daily_stats
# main_app = None # main_app = None
@ -112,22 +114,32 @@ class MonitoringData:
logger.info("워커 상태가 비어있어 기본값 사용") logger.info("워커 상태가 비어있어 기본값 사용")
worker_status = self._get_default_worker_status() worker_status = self._get_default_worker_status()
# 메인 서버와 프로세스가 분리되어 있으므로, status.json 값을 우선 사용 # 실시간 세션/워커 상태를 메인 서버 API에서 직접 가져오기
# 단, 같은 프로세스에서 실행되어 session_pool 이 초기화되어 있다면 실시간 값을 사용
try: try:
if getattr(session_pool, "_initialized", False): logger.info("실시간 세션/워커 상태 조회 시작")
real_session_status = session_pool.get_status() response = requests.get(f"http://{settings.HOST}:{settings.PORT}/api/v1/realtime_status", timeout=2)
if real_session_status: if response.status_code == 200:
session_status = real_session_status realtime_data = response.json()
logger.debug(f"실시간 세션 풀 상태 사용: {real_session_status}") if realtime_data.get("session_status"):
session_status = realtime_data["session_status"]
if not session_status: logger.info(f"✅ 실시간 세션 상태 조회 성공: {session_status}")
logger.info("세션 상태가 비어 status.json 값 또는 기본값 사용") if realtime_data.get("worker_status"):
session_status = status.get("session_status", {}) or self._get_default_session_status() worker_status = realtime_data["worker_status"]
except Exception as e: logger.info(f"✅ 실시간 워커 상태 조회 성공")
logger.warning(f"세션 풀 상태 조회 실패: {e}") else:
logger.warning(f"실시간 상태 조회 실패: 상태 코드 {response.status_code}")
except requests.RequestException as e:
logger.warning(f"실시간 상태 조회 중 예외 발생: {e}")
# 실시간 조회 실패 시 status.json 폴백
if not session_status:
logger.info("실시간 조회 실패, status.json 값 사용")
session_status = status.get("session_status", {}) or self._get_default_session_status() session_status = status.get("session_status", {}) or self._get_default_session_status()
if not worker_status:
logger.info("실시간 워커 조회 실패, status.json 값 사용")
worker_status = status.get("worker_status", {}) or self._get_default_worker_status()
# GPU 정보 (안전하게 가져오기) # GPU 정보 (안전하게 가져오기)
gpu_info = {} gpu_info = {}
try: try:
@ -237,19 +249,27 @@ class MonitoringData:
alerts = [] alerts = []
logger.info("데이터 구조 생성 시작") logger.info("데이터 구조 생성 시작")
# 세션/워커 이벤트 가져오기 (실시간 반영용)
session_events = []
try:
session_events = get_recent_events(limit=100)
except Exception as e:
logger.warning(f"세션 이벤트 조회 실패: {e}")
data = { data = {
"timestamp": datetime.now().isoformat(), "timestamp": datetime.now().isoformat(),
"system_type": "Jetson Xavier" if settings.IS_JETSON else "x86_64", "system_type": "Jetson Xavier" if settings.IS_JETSON else "x86_64",
"gpu": gpu_info, "gpu": gpu_info,
"system_memory": system_memory, "system_memory": system_memory,
"system_performance": system_performance, "system_performance": system_performance,
# status.json 스냅샷 외에 실시간 상태를 병합 "workers": worker_status,
"workers": worker_manager.get_status() or worker_status, "sessions": session_status,
"sessions": session_pool.get_status() or session_status,
"jetson": jetson_info, "jetson": jetson_info,
"api_stats": api_stats, "api_stats": api_stats,
"model_performance_stats": model_performance_stats, "model_performance_stats": model_performance_stats,
"alerts": alerts "alerts": alerts,
"session_events": session_events
} }
logger.info("히스토리에 데이터 추가 시작") logger.info("히스토리에 데이터 추가 시작")
@ -721,6 +741,31 @@ HTML_TEMPLATE = """
50% { opacity: 0.5; } 50% { opacity: 0.5; }
100% { opacity: 1; } 100% { opacity: 1; }
} }
.stat-item {
padding: 15px;
background: #f8f9fa;
border-radius: 8px;
text-align: center;
}
.stat-label {
font-size: 0.9em;
color: #666;
margin-bottom: 8px;
}
.stat-value {
font-size: 1.8em;
font-weight: bold;
color: #667eea;
margin-bottom: 5px;
}
.stat-details {
font-size: 0.85em;
color: #999;
}
</style> </style>
</head> </head>
<body> <body>
@ -985,17 +1030,6 @@ HTML_TEMPLATE = """
</div> </div>
</div> </div>
<!-- 스케일/세션 타임라인 -->
<div class="card">
<h3>📈 워커·세션 타임라인</h3>
<div id="scale-timeline" style="font-family:'Courier New',monospace;font-size:12px;background:#f8f9fa;border-radius:6px;padding:10px;max-height:220px;overflow:auto;">
로딩 ...
</div>
<div style="margin-top:8px;text-align:right;">
<button onclick="refreshTimeline()" style="padding:5px 12px;">새로고침</button>
</div>
</div>
<!-- 최근 에러 --> <!-- 최근 에러 -->
<div class="card"> <div class="card">
<h3>🚨 최근 API 에러</h3> <h3>🚨 최근 API 에러</h3>
@ -1024,6 +1058,41 @@ HTML_TEMPLATE = """
</div> </div>
</div> </div>
<!-- 일일 통계 -->
<div class="card">
<h3>📊 오늘의 통계</h3>
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 15px;">
<div class="stat-item">
<div class="stat-label">처리한 이미지</div>
<div class="stat-value" id="daily-images-total">-</div>
<div class="stat-details" id="daily-images-breakdown">-</div>
</div>
<div class="stat-item">
<div class="stat-label">업로드 용량</div>
<div class="stat-value" id="daily-upload-size">-</div>
</div>
<div class="stat-item">
<div class="stat-label">다운로드 용량</div>
<div class="stat-value" id="daily-download-size">-</div>
</div>
<div class="stat-item">
<div class="stat-label">최대 동시 요청</div>
<div class="stat-value" id="daily-peak-concurrent">-</div>
</div>
</div>
</div>
<!-- 세션/워커 타임라인 -->
<div class="card">
<h3>🔄 세션·워커 타임라인</h3>
<div class="timeline-container" id="timeline-container" style="max-height: 300px; overflow-y: auto; border: 1px solid #eee; border-radius: 5px; padding: 10px;">
<div id="timeline-body">로딩 ...</div>
</div>
<div style="margin-top: 10px; text-align: right;">
<button onclick="refreshTimeline()" style="padding: 5px 15px; background: #667eea; color: white; border: none; border-radius: 3px; cursor: pointer;">새로고침</button>
</div>
</div>
<!-- 모델 처리 시간 통계 --> <!-- 모델 처리 시간 통계 -->
<div class="card"> <div class="card">
<h3> 모델 처리 시간 통계 ()</h3> <h3> 모델 처리 시간 통계 ()</h3>
@ -1058,6 +1127,16 @@ HTML_TEMPLATE = """
<canvas id="gpuChart" width="400" height="200"></canvas> <canvas id="gpuChart" width="400" height="200"></canvas>
</div> </div>
<div class="chart-container">
<h3>🔧 세션 사용량</h3>
<canvas id="sessionChart" width="400" height="200"></canvas>
</div>
<div class="chart-container">
<h3> 워커 활성도</h3>
<canvas id="workerChart" width="400" height="200"></canvas>
</div>
<div class="refresh-time"> <div class="refresh-time">
마지막 업데이트: <span id="last-update">-</span> | 마지막 업데이트: <span id="last-update">-</span> |
연결 상태: <span id="connection-status" class="status connecting">연결 ...</span> 연결 상태: <span id="connection-status" class="status connecting">연결 ...</span>
@ -1129,8 +1208,7 @@ HTML_TEMPLATE = """
function connectWebSocket() { function connectWebSocket() {
try { try {
const proto = (window.location.protocol === 'https:') ? 'wss' : 'ws'; ws = new WebSocket(`ws://${window.location.host}/ws`);
ws = new WebSocket(`${proto}://${window.location.host}/ws`);
ws.onopen = function() { ws.onopen = function() {
console.log('WebSocket 연결이 성공했습니다.'); console.log('WebSocket 연결이 성공했습니다.');
@ -1150,6 +1228,11 @@ HTML_TEMPLATE = """
} }
updateDashboard(data); updateDashboard(data);
// 세션 이벤트가 있으면 타임라인 업데이트
if (data.session_events) {
renderTimeline(data.session_events);
}
} catch (e) { } catch (e) {
console.error('데이터 파싱 오류:', e); console.error('데이터 파싱 오류:', e);
} }
@ -1446,6 +1529,71 @@ HTML_TEMPLATE = """
}); });
} }
function refreshTimeline() {
fetch('/api/session_events?limit=100')
.then(r => r.json())
.then(data => renderTimeline(data.events || []))
.catch(() => {
document.getElementById('timeline-body').innerHTML = '<div style="color:#dc3545; padding:8px;">타임라인 로드 실패</div>';
});
}
function renderTimeline(events) {
const container = document.getElementById('timeline-body');
if (!events || events.length === 0) {
container.innerHTML = '<div style="padding:8px; color:#999;">이벤트 없음</div>';
return;
}
// 최신 이벤트가 위로 오도록 역순 정렬
const sortedEvents = events.slice().sort((a, b) => b.timestamp - a.timestamp);
let html = '<div style="font-family: monospace; font-size: 13px;">';
sortedEvents.forEach(event => {
const timestamp = new Date(event.timestamp * 1000).toLocaleString('ko-KR', {
year: '2-digit',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
second: '2-digit'
});
let icon = '🔹';
let color = '#6c757d';
if (event.event_type === 'session_create') {
icon = '';
color = '#28a745';
} else if (event.event_type === 'session_destroy') {
icon = '';
color = '#dc3545';
} else if (event.event_type.includes('scale_up')) {
icon = '⬆️';
color = '#007bff';
} else if (event.event_type.includes('scale_down')) {
icon = '⬇️';
color = '#ffc107';
}
html += '<div style="padding: 6px; border-bottom: 1px solid #f0f0f0; display: flex; align-items: center;">';
html += '<span style="margin-right: 8px;">' + icon + '</span>';
html += '<span style="color: #999; min-width: 140px;">' + timestamp + '</span>';
html += '<span style="color: ' + color + '; font-weight: bold; margin-right: 8px;">' + event.event_type + '</span>';
if (event.model_type) {
html += '<span style="color: #667eea; margin-right: 8px;">[' + event.model_type + ']</span>';
}
if (event.session_id) {
html += '<span style="color: #6c757d;">' + event.session_id + '</span>';
}
if (event.details && Object.keys(event.details).length > 0) {
html += '<span style="color: #999; margin-left: 8px; font-size: 11px;">' + JSON.stringify(event.details) + '</span>';
}
html += '</div>';
});
html += '</div>';
container.innerHTML = html;
}
function updateAlerts(alerts) { function updateAlerts(alerts) {
const container = document.getElementById('alerts-container'); const container = document.getElementById('alerts-container');
container.innerHTML = ''; container.innerHTML = '';
@ -1719,32 +1867,6 @@ HTML_TEMPLATE = """
}); });
} }
function renderTimeline(events) {
const el = document.getElementById('scale-timeline');
if (!Array.isArray(events) || events.length === 0) {
el.innerHTML = '<div style="color:#999;">이벤트 없음</div>';
return;
}
const rows = events.slice().reverse().map(ev => {
const ts = ev.timestamp ? new Date(ev.timestamp*1000).toLocaleTimeString() : '';
if (ev.type === 'worker_scale') {
return `[${ts}] WORKERS ${ev.action.toUpperCase()} -> ${ev.new_count} (queue=${ev.queue_size}, vram=${(ev.vram_usage*100||0).toFixed(1)}%)`;
}
if (ev.type === 'session') {
return `[${ts}] SESSION ${ev.action.toUpperCase()} (${ev.model}) size=${ev.pool_size}`;
}
return `[${ts}] ${ev.type}`;
}).join('\n');
el.textContent = rows;
}
function refreshTimeline() {
fetch('/api/scale-events')
.then(r => r.json())
.then(data => renderTimeline(data.events || []))
.catch(() => { document.getElementById('scale-timeline').innerHTML = '<div style="color:#dc3545;">타임라인 로딩 실패</div>'; });
}
// 페이지 로드 초기화 // 페이지 로드 초기화
document.addEventListener('DOMContentLoaded', function() { document.addEventListener('DOMContentLoaded', function() {
// 로그 성능 통계 초기 로딩 // 로그 성능 통계 초기 로딩
@ -1753,6 +1875,7 @@ HTML_TEMPLATE = """
refreshModelUsageStats(); refreshModelUsageStats();
refreshSystemAlerts(); refreshSystemAlerts();
refreshErrors(); refreshErrors();
refreshTimeline();
// 30초마다 로그 성능 통계 자동 새로고침 // 30초마다 로그 성능 통계 자동 새로고침
setInterval(refreshLogs, 30000); setInterval(refreshLogs, 30000);
@ -1760,6 +1883,7 @@ HTML_TEMPLATE = """
setInterval(refreshModelUsageStats, 15000); // 15초마다 setInterval(refreshModelUsageStats, 15000); // 15초마다
setInterval(refreshSystemAlerts, 10000); // 10초마다 setInterval(refreshSystemAlerts, 10000); // 10초마다
setInterval(refreshErrors, 10000); // 10초마다 setInterval(refreshErrors, 10000); // 10초마다
setInterval(refreshTimeline, 15000); // 15초마다 타임라인 새로고침
}); });
</script> </script>
</body> </body>
@ -2054,16 +2178,6 @@ async def get_system_alerts():
logger.error(f"시스템 알림 조회 실패: {e}") logger.error(f"시스템 알림 조회 실패: {e}")
return {"alerts": [], "error": str(e)} return {"alerts": [], "error": str(e)}
@api_router.get("/scale-events")
def get_scale_events():
"""최근 스케일/세션 이벤트를 반환"""
try:
from ..utils.monitor_events import read_recent_events
return {"events": read_recent_events(limit=300)}
except Exception as e:
logger.error(f"타임라인 조회 실패: {e}")
return {"events": [], "error": str(e)}
@api_router.get("/errors", summary="최근 API 에러 목록") @api_router.get("/errors", summary="최근 API 에러 목록")
def get_recent_errors(limit: int = 50): def get_recent_errors(limit: int = 50):
"""최근 API 에러를 반환합니다 (logs/api_errors.jsonl 기반).""" """최근 API 에러를 반환합니다 (logs/api_errors.jsonl 기반)."""
@ -2073,6 +2187,16 @@ def get_recent_errors(limit: int = 50):
logger.error(f"에러 목록 조회 실패: {e}") logger.error(f"에러 목록 조회 실패: {e}")
return {"errors": [], "error": str(e)} return {"errors": [], "error": str(e)}
@api_router.get("/session_events", summary="최근 세션/워커 이벤트")
def get_session_events(limit: int = 100):
"""최근 세션/워커 생성·해제·스케일 이벤트를 반환합니다."""
try:
events = get_recent_events(limit=limit)
return {"events": events}
except Exception as e:
logger.error(f"세션 이벤트 조회 실패: {e}")
return {"events": [], "error": str(e)}
@api_router.get("/test") @api_router.get("/test")
async def test_endpoint(): async def test_endpoint():
"""테스트용 엔드포인트입니다.""" """테스트용 엔드포인트입니다."""
@ -2347,29 +2471,3 @@ if __name__ == "__main__":
port=settings.MONITORING_PORT, port=settings.MONITORING_PORT,
log_level="info" log_level="info"
) )
# --- 외부 런처용: 로그에 시간 포함하여 실행 ---
def _get_uvicorn_log_config():
try:
from uvicorn.config import LOGGING_CONFIG as DEFAULT
import copy
cfg = copy.deepcopy(DEFAULT)
# 포맷에 시간 추가
for fmt in ("default", "access"):
if fmt in cfg.get("formatters", {}):
cfg["formatters"][fmt]["format"] = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
return cfg
except Exception:
return None
def run_monitor(host: str = "0.0.0.0", port: int = None):
"""모니터링 서버 실행 (시간 스탬프 포함 로그)"""
_port = port or settings.MONITORING_PORT
uvicorn.run(
monitor_app,
host=host,
port=_port,
log_level="info",
log_config=_get_uvicorn_log_config()
)

203
app/utils/daily_stats.py Normal file
View File

@ -0,0 +1,203 @@
"""
일일 통계 수집 관리
- 처리된 이미지
- 네트워크 전송량
- API 호출 통계
"""
from __future__ import annotations
import os
import time
import json
from datetime import datetime, timedelta
from typing import Dict, Any
from threading import Lock
from collections import defaultdict
LOG_DIR = "logs"
os.makedirs(LOG_DIR, exist_ok=True)
DAILY_STATS_PATH = os.path.join(LOG_DIR, "daily_stats.json")
class DailyStatsCollector:
def __init__(self):
self.lock = Lock()
self.current_date = datetime.now().strftime("%Y-%m-%d")
self.stats = self._load_or_create_today_stats()
def _load_or_create_today_stats(self) -> Dict[str, Any]:
"""오늘의 통계를 로드하거나 새로 생성"""
today = datetime.now().strftime("%Y-%m-%d")
try:
if os.path.exists(DAILY_STATS_PATH):
with open(DAILY_STATS_PATH, "r", encoding="utf-8") as f:
all_stats = json.load(f)
# 오늘 날짜의 통계가 있으면 반환
if today in all_stats:
return all_stats[today]
except Exception:
pass
# 새로운 통계 생성
return {
"date": today,
"images_processed": {
"inpaint": 0,
"remove_bg": 0,
"gen_image": 0,
"total": 0
},
"network": {
"bytes_uploaded": 0,
"bytes_downloaded": 0,
"requests_count": 0
},
"api_calls": {
"total": 0,
"success": 0,
"failed": 0
},
"models_used": defaultdict(int),
"peak_concurrent": 0,
"start_time": time.time(),
"last_update": time.time()
}
def _check_date_rollover(self):
"""날짜가 바뀌면 통계 저장 및 리셋"""
today = datetime.now().strftime("%Y-%m-%d")
if today != self.current_date:
# 어제 통계 저장
self._save_stats()
# 새로운 날짜로 리셋
self.current_date = today
self.stats = self._load_or_create_today_stats()
def record_image_processed(self, endpoint_type: str):
"""이미지 처리 기록"""
with self.lock:
self._check_date_rollover()
if endpoint_type == "inpaint":
self.stats["images_processed"]["inpaint"] += 1
elif endpoint_type == "remove_bg":
self.stats["images_processed"]["remove_bg"] += 1
elif endpoint_type == "gen_image":
self.stats["images_processed"]["gen_image"] += 1
self.stats["images_processed"]["total"] += 1
self.stats["last_update"] = time.time()
def record_network_traffic(self, bytes_uploaded: int, bytes_downloaded: int):
"""네트워크 트래픽 기록"""
with self.lock:
self._check_date_rollover()
self.stats["network"]["bytes_uploaded"] += bytes_uploaded
self.stats["network"]["bytes_downloaded"] += bytes_downloaded
self.stats["network"]["requests_count"] += 1
self.stats["last_update"] = time.time()
def record_api_call(self, success: bool):
"""API 호출 기록"""
with self.lock:
self._check_date_rollover()
self.stats["api_calls"]["total"] += 1
if success:
self.stats["api_calls"]["success"] += 1
else:
self.stats["api_calls"]["failed"] += 1
self.stats["last_update"] = time.time()
def record_model_usage(self, model_name: str):
"""모델 사용 기록"""
with self.lock:
self._check_date_rollover()
if "models_used" not in self.stats:
self.stats["models_used"] = {}
self.stats["models_used"][model_name] = self.stats["models_used"].get(model_name, 0) + 1
self.stats["last_update"] = time.time()
def update_peak_concurrent(self, current_concurrent: int):
"""최대 동시 요청 수 업데이트"""
with self.lock:
self._check_date_rollover()
if current_concurrent > self.stats["peak_concurrent"]:
self.stats["peak_concurrent"] = current_concurrent
self.stats["last_update"] = time.time()
def get_today_stats(self) -> Dict[str, Any]:
"""오늘의 통계 반환"""
with self.lock:
self._check_date_rollover()
# 읽기 전용 복사본 반환
stats_copy = dict(self.stats)
# MB/GB 단위로 변환된 값 추가
stats_copy["network"]["mb_uploaded"] = stats_copy["network"]["bytes_uploaded"] / (1024 * 1024)
stats_copy["network"]["mb_downloaded"] = stats_copy["network"]["bytes_downloaded"] / (1024 * 1024)
stats_copy["network"]["gb_uploaded"] = stats_copy["network"]["bytes_uploaded"] / (1024 * 1024 * 1024)
stats_copy["network"]["gb_downloaded"] = stats_copy["network"]["bytes_downloaded"] / (1024 * 1024 * 1024)
return stats_copy
def get_historical_stats(self, days: int = 7) -> Dict[str, Any]:
"""최근 N일간의 통계 반환"""
try:
if not os.path.exists(DAILY_STATS_PATH):
return {}
with open(DAILY_STATS_PATH, "r", encoding="utf-8") as f:
all_stats = json.load(f)
# 최근 N일 필터링
cutoff_date = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
recent_stats = {
date: stats for date, stats in all_stats.items()
if date >= cutoff_date
}
return recent_stats
except Exception:
return {}
def _save_stats(self):
"""현재 통계를 파일에 저장"""
try:
# 기존 통계 로드
all_stats = {}
if os.path.exists(DAILY_STATS_PATH):
with open(DAILY_STATS_PATH, "r", encoding="utf-8") as f:
all_stats = json.load(f)
# 현재 날짜 통계 업데이트
all_stats[self.current_date] = self.stats
# 30일 이상 된 통계 삭제
cutoff_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
all_stats = {
date: stats for date, stats in all_stats.items()
if date >= cutoff_date
}
# 파일에 저장
with open(DAILY_STATS_PATH, "w", encoding="utf-8") as f:
json.dump(all_stats, f, indent=2, ensure_ascii=False)
except Exception:
pass
def save(self):
"""수동 저장"""
with self.lock:
self._save_stats()
# 글로벌 인스턴스
daily_stats = DailyStatsCollector()

View File

@ -1,69 +0,0 @@
"""
경량 모니터링 이벤트(JSONL) 기록 읽기 유틸
- worker 스케일 /다운
- 세션 생성/회수
"""
from __future__ import annotations
import os
import time
import json
from typing import Dict, Any, List
LOG_DIR = "logs"
os.makedirs(LOG_DIR, exist_ok=True)
EVENT_LOG_PATH = os.path.join(LOG_DIR, "scale_events.jsonl")
MAX_BYTES = 10 * 1024 * 1024 # 10MB
BACKUP = 10
def _rotate_if_needed():
try:
if os.path.exists(EVENT_LOG_PATH) and os.path.getsize(EVENT_LOG_PATH) > MAX_BYTES:
ts = time.strftime("%Y%m%d-%H%M%S")
os.replace(EVENT_LOG_PATH, os.path.join(LOG_DIR, f"scale_events_{ts}.jsonl"))
rotated = [os.path.join(LOG_DIR, f) for f in os.listdir(LOG_DIR) if f.startswith("scale_events_")]
rotated.sort(key=lambda p: os.path.getmtime(p), reverse=True)
for p in rotated[BACKUP:]:
try:
os.remove(p)
except Exception:
pass
except Exception:
pass
def append_event(event: Dict[str, Any]) -> None:
try:
_rotate_if_needed()
if "timestamp" not in event:
event["timestamp"] = time.time()
with open(EVENT_LOG_PATH, "a", encoding="utf-8") as f:
f.write(json.dumps(event, ensure_ascii=False) + "\n")
except Exception:
pass
def read_recent_events(limit: int = 300) -> List[Dict[str, Any]]:
try:
if not os.path.exists(EVENT_LOG_PATH):
return []
events: List[Dict[str, Any]] = []
with open(EVENT_LOG_PATH, "r", encoding="utf-8") as f:
# 간단히 끝에서 limit줄만 읽기 (파일이 크지 않다고 가정)
lines = f.readlines()[-limit:]
for line in lines:
line = line.strip()
if not line:
continue
try:
events.append(json.loads(line))
except Exception:
continue
return events
except Exception:
return []

View File

@ -0,0 +1,101 @@
"""
세션/워커 생성·해제·스케일 이벤트를 기록하고 대시보드로 전달
"""
from __future__ import annotations
import os
import time
import json
from collections import deque
from typing import Dict, Any, List
from threading import Lock
LOG_DIR = "logs"
os.makedirs(LOG_DIR, exist_ok=True)
SESSION_EVENT_LOG_PATH = os.path.join(LOG_DIR, "session_events.jsonl")
SESSION_EVENT_MAX_BYTES = 5 * 1024 * 1024 # 5MB
SESSION_EVENT_BACKUP_COUNT = 3
# 메모리 내 최근 이벤트 버퍼 (대시보드 실시간 전송용)
_recent_events: deque = deque(maxlen=200)
_events_lock = Lock()
def _rotate_if_needed() -> None:
try:
if os.path.exists(SESSION_EVENT_LOG_PATH) and os.path.getsize(SESSION_EVENT_LOG_PATH) >= SESSION_EVENT_MAX_BYTES:
ts = time.strftime("%Y%m%d-%H%M%S")
rotated_path = os.path.join(LOG_DIR, f"session_events_{ts}.jsonl")
os.replace(SESSION_EVENT_LOG_PATH, rotated_path)
rotated = [
os.path.join(LOG_DIR, f) for f in os.listdir(LOG_DIR)
if f.startswith("session_events_") and f.endswith(".jsonl")
]
rotated.sort(key=lambda p: os.path.getmtime(p), reverse=True)
for old in rotated[SESSION_EVENT_BACKUP_COUNT:]:
try:
os.remove(old)
except Exception:
pass
except Exception:
pass
def log_session_event(
event_type: str, # "session_create", "session_destroy", "worker_scale_up", "worker_scale_down", "pool_reap", etc.
model_type: str = "",
session_id: str = "",
details: Dict[str, Any] | None = None
) -> None:
"""
세션/워커 이벤트 기록
- JSONL 파일로 영구 저장 (로테이션)
- 메모리 버퍼에 최근 이벤트 유지 (대시보드 실시간 전송)
"""
record = {
"timestamp": time.time(),
"event_type": event_type,
"model_type": model_type,
"session_id": session_id,
"details": details or {}
}
try:
_rotate_if_needed()
with open(SESSION_EVENT_LOG_PATH, "a", encoding="utf-8") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
except Exception:
pass
with _events_lock:
_recent_events.append(record)
def get_recent_events(limit: int = 100) -> List[Dict[str, Any]]:
"""최근 이벤트 반환 (대시보드 API용)"""
with _events_lock:
return list(_recent_events)[-limit:]
def read_events_from_file(limit: int = 200) -> List[Dict[str, Any]]:
"""파일에서 최근 이벤트 읽기 (초기 로드용)"""
events = []
try:
if not os.path.exists(SESSION_EVENT_LOG_PATH):
return events
with open(SESSION_EVENT_LOG_PATH, "r", encoding="utf-8") as f:
lines = f.readlines()
for line in lines[-limit:]:
try:
events.append(json.loads(line.strip()))
except Exception:
pass
except Exception:
pass
return events

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1 +1 @@
271615 326469

View File

@ -1,69 +1,49 @@
WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats' WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats'
INFO: Started server process [271803] INFO: Started server process [326863]
INFO: Waiting for application startup. INFO: Waiting for application startup.
INFO: Application startup complete. INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:8888 (Press CTRL+C to quit) INFO: Uvicorn running on http://0.0.0.0:8888 (Press CTRL+C to quit)
INFO: 127.0.0.1:49994 - "GET /api/simple HTTP/1.1" 200 OK INFO: 122.35.47.45:59571 - "WebSocket /ws" [accepted]
INFO: 118.235.73.64:35921 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 118.235.73.64:35815 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:27910 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:33163 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:33780 - "WebSocket /ws" [accepted]
INFO: connection open
INFO: 118.235.73.64:29039 - "GET /favicon.ico HTTP/1.1" 404 Not Found
ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: connection closed
INFO: 118.235.73.64:36365 - "WebSocket /ws" [accepted]
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2) ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: connection open INFO: connection open
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2) WARNING:app.monitoring.dashboard:실시간 상태 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 118.235.73.64:29969 - "GET /api/system-alerts HTTP/1.1" 200 OK INFO: 122.35.47.45:59569 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:36188 - "GET /api/errors HTTP/1.1" 200 OK INFO: 122.35.47.45:59567 - "GET / HTTP/1.1" 200 OK
INFO: 118.235.73.64:34776 - "GET /api/model-usage-stats HTTP/1.1" 200 OK INFO: 122.35.47.45:59574 - "GET / HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2) INFO: 122.35.47.45:59575 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:34776 - "GET /api/system-alerts HTTP/1.1" 200 OK INFO: 127.0.0.1:56324 - "GET /api/simple HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2) INFO: 122.35.47.45:59578 - "GET / HTTP/1.1" 200 OK
INFO: 118.235.73.64:36188 - "GET /api/errors HTTP/1.1" 200 OK INFO: 122.35.47.45:59569 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:52268 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59577 - "WebSocket /ws" [accepted]
INFO: 118.235.73.64:36188 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK INFO: connection open
INFO: 118.235.73.64:32011 - "GET /api/model-usage-stats HTTP/1.1" 200 OK INFO: 122.35.47.45:59576 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 118.235.73.64:29864 - "GET /api/performance-stats HTTP/1.1" 200 OK INFO: 122.35.47.45:59568 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 118.235.73.64:33588 - "GET /api/system-alerts HTTP/1.1" 200 OK INFO: 122.35.47.45:59578 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 118.235.73.64:31627 - "GET /api/errors HTTP/1.1" 200 OK INFO: 122.35.47.45:59575 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:32554 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:36432 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 118.235.73.64:36432 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:36432 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:31023 - "GET /api/errors HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 118.235.73.64:30785 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:35188 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:30117 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:30102 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 118.235.73.64:36300 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 118.235.73.64:30102 - "GET / HTTP/1.1" 200 OK
INFO: connection closed
INFO: 118.235.73.64:30102 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 118.235.73.64:34126 - "GET /favicon.ico HTTP/1.1" 404 Not Found
ERROR:app.monitoring.dashboard:데이터 전송 오류: ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: 118.235.73.64:27884 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59584 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
WARNING: Invalid HTTP request received. INFO: connection closed
WARNING: Invalid HTTP request received. INFO: connection closed
INFO: 118.235.73.64:30128 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59592 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:30128 - "GET /favicon.ico HTTP/1.1" 404 Not Found INFO: 122.35.47.45:59575 - "GET / HTTP/1.1" 200 OK
INFO: 118.235.73.64:33038 - "GET /favicon.ico HTTP/1.1" 404 Not Found ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: 118.235.73.64:34281 - "GET /favicon.ico HTTP/1.1" 404 Not Found INFO: 122.35.47.45:59588 - "WebSocket /ws" [accepted]
INFO: 118.235.73.64:28205 - "GET / HTTP/1.1" 200 OK INFO: connection open
INFO: 118.235.73.64:28205 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59592 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:30339 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59575 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 118.235.73.64:30775 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59604 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 118.235.73.64:32870 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59603 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 118.235.73.64:28161 - "GET / HTTP/1.1" 200 OK ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: 118.235.73.64:33203 - "GET / HTTP/1.1" 200 OK INFO: connection closed
INFO: 118.235.73.64:37097 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59578 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 118.235.73.64:36624 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59602 - "WebSocket /ws" [accepted]
INFO: 118.235.73.64:30357 - "GET / HTTP/1.1" 200 OK INFO: connection open
INFO: 118.235.73.64:35740 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:59584 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:50214 - "GET / HTTP/1.1" 200 OK ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 122.35.47.45:50214 - "GET / HTTP/1.1" 200 OK WARNING:app.monitoring.dashboard:실시간 상태 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 122.35.47.45:59624 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59625 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:59630 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:59629 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:59629 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59630 - "GET /api/errors HTTP/1.1" 200 OK

View File

@ -1 +1 @@
271803 326863

22
main.py
View File

@ -28,6 +28,7 @@ from app.utils.api_error_log import (
extract_client_ip, extract_client_ip,
get_content_length, get_content_length,
) )
from app.utils.daily_stats import daily_stats
# 로깅 설정 # 로깅 설정
import logging.handlers import logging.handlers
@ -239,10 +240,14 @@ async def save_status_periodically():
api_statistics = api_stats.get_stats() api_statistics = api_stats.get_stats()
logger.debug(f"API 통계 수집 완료: 총 요청 {api_statistics['total_requests']}") logger.debug(f"API 통계 수집 완료: 총 요청 {api_statistics['total_requests']}")
# 일일 통계 수집
today_stats = daily_stats.get_today_stats()
status = { status = {
"worker_status": worker_status, "worker_status": worker_status,
"session_status": session_status, "session_status": session_status,
"api_stats": api_statistics, "api_stats": api_statistics,
"daily_stats": today_stats,
"timestamp": time.time() "timestamp": time.time()
} }
@ -392,6 +397,23 @@ async def collect_api_stats(request: Request, call_next):
# 통계 업데이트 # 통계 업데이트
api_stats.end_request(endpoint, success, response_time) api_stats.end_request(endpoint, success, response_time)
# 일일 통계 기록
daily_stats.record_api_call(success)
daily_stats.update_peak_concurrent(api_stats.current_concurrent)
# 이미지 처리 엔드포인트 기록
if path == "/api/v1/inpaint":
daily_stats.record_image_processed("inpaint")
elif path == "/api/v1/remove_bg":
daily_stats.record_image_processed("remove_bg")
elif path == "/api/v1/run_plugin_gen_image":
daily_stats.record_image_processed("gen_image")
# 네트워크 트래픽 기록 (대략적인 추정)
content_length = get_content_length(request)
response_size = int(response.headers.get("content-length", 0))
daily_stats.record_network_traffic(content_length, response_size)
# 4xx/5xx는 에러 로그 파일에 기록 (클라이언트 IP 포함) # 4xx/5xx는 에러 로그 파일에 기록 (클라이언트 IP 포함)
if not success: if not success:

219
scripts/start_monitoring.sh Executable file
View File

@ -0,0 +1,219 @@
#!/bin/bash
# 모니터링 대시보드 시작 스크립트
# Usage: ./start_monitoring.sh
set -e
# 색상 코드
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# 로그 함수들
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# 기본 설정 - 동적 경로 처리
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
MONITORING_PORT=8888
LOG_DIR="$PROJECT_ROOT/logs"
# 가상환경 경로 자동 감지
detect_venv_path() {
local possible_paths=(
"$PROJECT_ROOT/venv" # 일반적인 venv 경로
"$PROJECT_ROOT/.venv" # 숨김 venv 경로
"$PROJECT_ROOT/env" # env 경로
"$PROJECT_ROOT/.env" # 숨김 env 경로
"$PROJECT_ROOT" # 프로젝트 루트 (Jetson 방식)
)
for path in "${possible_paths[@]}"; do
if [ -f "$path/bin/python" ] || [ -f "$path/bin/python3" ]; then
VENV_PATH="$path"
log_info "가상환경 경로 감지: $VENV_PATH"
return 0
fi
done
# 현재 활성화된 가상환경 확인
if [ -n "$VIRTUAL_ENV" ]; then
VENV_PATH="$VIRTUAL_ENV"
log_info "활성화된 가상환경 사용: $VENV_PATH"
return 0
fi
# 가상환경을 찾지 못한 경우 시스템별 기본값 사용
if [ "$(uname -m)" = "aarch64" ] && uname -a | grep -q "tegra"; then
VENV_PATH="$PROJECT_ROOT"
log_warning "가상환경을 찾을 수 없어 Jetson 기본값 사용: $VENV_PATH"
else
VENV_PATH="$PROJECT_ROOT/venv"
log_warning "가상환경을 찾을 수 없어 x86 기본값 사용: $VENV_PATH"
fi
}
# 기존 모니터링 서버 중지
stop_existing_monitoring() {
log_info "기존 모니터링 서버 확인 중..."
# PID 파일이 있으면 해당 프로세스 종료
if [ -f "$LOG_DIR/monitoring.pid" ]; then
local old_pid=$(cat "$LOG_DIR/monitoring.pid")
if ps -p "$old_pid" > /dev/null 2>&1; then
log_info "기존 모니터링 서버 종료 중... (PID: $old_pid)"
kill "$old_pid" 2>/dev/null || true
sleep 2
# 강제 종료가 필요한 경우
if ps -p "$old_pid" > /dev/null 2>&1; then
log_warning "강제 종료 시도 중..."
kill -9 "$old_pid" 2>/dev/null || true
sleep 1
fi
fi
rm -f "$LOG_DIR/monitoring.pid"
fi
# 포트를 사용하는 프로세스 확인 및 종료
if lsof -Pi :$MONITORING_PORT -sTCP:LISTEN -t >/dev/null 2>&1; then
log_warning "모니터링 포트 $MONITORING_PORT가 이미 사용 중입니다"
log_info "포트를 사용하는 프로세스를 종료합니다..."
# 프로세스 이름으로 종료 시도
pkill -f "dashboard:monitor_app" 2>/dev/null || true
pkill -f "monitoring.dashboard" 2>/dev/null || true
sleep 2
# 여전히 포트를 사용 중이면 강제 종료
if lsof -Pi :$MONITORING_PORT -sTCP:LISTEN -t >/dev/null 2>&1; then
log_warning "포트를 강제로 해제합니다..."
lsof -ti:$MONITORING_PORT | xargs -r kill -9 2>/dev/null || true
sleep 2
fi
# 최종 확인
if lsof -Pi :$MONITORING_PORT -sTCP:LISTEN -t >/dev/null 2>&1; then
log_error "포트 $MONITORING_PORT를 해제할 수 없습니다"
lsof -Pi :$MONITORING_PORT -sTCP:LISTEN
exit 1
fi
fi
log_success "기존 모니터링 서버 정리 완료"
}
# 가상환경 활성화
activate_venv() {
log_info "가상환경 활성화 중..."
cd "$PROJECT_ROOT"
if [ -f "$VENV_PATH/bin/activate" ]; then
source "$VENV_PATH/bin/activate"
log_success "가상환경 활성화 완료"
else
log_error "가상환경을 찾을 수 없습니다: $VENV_PATH/bin/activate"
exit 1
fi
# Python 버전 확인
PYTHON_VERSION=$(python --version 2>&1)
log_info "Python 버전: $PYTHON_VERSION"
}
# 모니터링 서버 시작
start_monitoring_server() {
log_info "모니터링 대시보드 시작 (포트: $MONITORING_PORT)..."
# 로그 디렉토리 생성
mkdir -p "$LOG_DIR"
# Python 경로 설정으로 import 문제 해결
cd "$PROJECT_ROOT"
PYTHONPATH="$PROJECT_ROOT" nohup python -c "
from app.monitoring.dashboard import monitor_app
import uvicorn
uvicorn.run(monitor_app, host='0.0.0.0', port=$MONITORING_PORT, log_level='info')
" > "$LOG_DIR/monitoring.log" 2>&1 &
echo $! > "$LOG_DIR/monitoring.pid"
# 서버 시작 대기
log_info "모니터링 서버 초기화 대기 중... (최대 10초)"
local timeout=10
local elapsed=0
local interval=1
while [ $elapsed -lt $timeout ]; do
sleep $interval
elapsed=$((elapsed + interval))
# 헬스체크
if curl -s "http://localhost:$MONITORING_PORT/api/simple" > /dev/null 2>&1; then
log_success "모니터링 대시보드 시작 완료 (PID: $(cat $LOG_DIR/monitoring.pid)) - ${elapsed}초 소요"
return 0
fi
# 진행 상황 로깅
if [ $((elapsed % 3)) -eq 0 ]; then
log_info "모니터링 서버 초기화 중... (${elapsed}/${timeout}초)"
fi
done
# 최종 확인
if ! curl -s "http://localhost:$MONITORING_PORT/api/simple" > /dev/null 2>&1; then
log_error "모니터링 서버 시작 실패 (${timeout}초 타임아웃)"
log_info "로그를 확인하세요: tail -f $LOG_DIR/monitoring.log"
exit 1
fi
}
# 상태 정보 출력
print_status() {
echo ""
echo "=========================================="
echo "📊 모니터링 대시보드 시작 완료!"
echo "=========================================="
echo "모니터링 URL: http://localhost:$MONITORING_PORT"
echo "API 엔드포인트: http://localhost:$MONITORING_PORT/api/simple"
echo ""
echo "PID: $(cat $LOG_DIR/monitoring.pid)"
echo "로그 파일: $LOG_DIR/monitoring.log"
echo ""
echo "서버 중지: kill \$(cat $LOG_DIR/monitoring.pid)"
echo "로그 확인: tail -f $LOG_DIR/monitoring.log"
echo "=========================================="
}
# 메인 실행
main() {
log_info "모니터링 대시보드 시작 스크립트 실행"
detect_venv_path
stop_existing_monitoring
activate_venv
start_monitoring_server
print_status
log_success "모니터링 대시보드가 성공적으로 시작되었습니다!"
}
# 스크립트 실행
main "$@"

View File

@ -6,42 +6,42 @@
"workers_by_status": { "workers_by_status": {
"idle": [ "idle": [
{ {
"id": "worker_3e3d5864", "id": "worker_0fc41239",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
"last_task_at": null "last_task_at": null
}, },
{ {
"id": "worker_7373b97c", "id": "worker_53a5b48f",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
"last_task_at": null "last_task_at": null
}, },
{ {
"id": "worker_296ab3a4", "id": "worker_c8680a6b",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
"last_task_at": null "last_task_at": null
}, },
{ {
"id": "worker_891dcc94", "id": "worker_dd334383",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
"last_task_at": null "last_task_at": null
}, },
{ {
"id": "worker_411ad2e8", "id": "worker_11d7c890",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
"last_task_at": null "last_task_at": null
}, },
{ {
"id": "worker_ab487451", "id": "worker_2dac3e33",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
@ -57,57 +57,65 @@
"session_status": { "session_status": {
"simple_lama": { "simple_lama": {
"min": 4, "min": 4,
"max": 8, "max": 6,
"total": 4, "total": 4,
"in_use": 0, "in_use": 0,
"available": 4 "available": 4
}, },
"migan": { "migan": {
"min": 1, "min": 1,
"max": 8, "max": 6,
"total": 1, "total": 1,
"in_use": 0, "in_use": 0,
"available": 1 "available": 1
}, },
"rembg": { "rembg": {
"min": 1, "min": 2,
"max": 4, "max": 6,
"total": 1, "total": 2,
"in_use": 0, "in_use": 0,
"available": 1 "available": 2
} }
}, },
"api_stats": { "api_stats": {
"total_requests": 13467, "total_requests": 95,
"successful_requests": 13467, "successful_requests": 95,
"failed_requests": 0, "failed_requests": 0,
"success_rate": 100.0, "success_rate": 100.0,
"endpoint_usage": { "endpoint_usage": {
"GET /api/v1/model": 6740, "GET /api/v1/model": 44,
"POST /api/v1/inpaint": 6264, "POST /api/v1/run_plugin_gen_image": 5,
"POST /api/v1/run_plugin_gen_image": 463 "POST /api/v1/inpaint": 26,
"GET /api/v1/realtime_status": 20
}, },
"endpoint_stats": { "endpoint_stats": {
"GET /api/v1/model": { "GET /api/v1/model": {
"count": 6740, "count": 44,
"avg_time": 0.001540846824645996, "avg_time": 0.00349293513731523,
"min_time": 0.00063323974609375, "min_time": 0.0007824897766113281,
"max_time": 0.004244089126586914, "max_time": 0.008572578430175781,
"current_concurrent": 0
},
"POST /api/v1/run_plugin_gen_image": {
"count": 5,
"avg_time": 1.928318691253662,
"min_time": 0.2756519317626953,
"max_time": 8.236981391906738,
"current_concurrent": 0 "current_concurrent": 0
}, },
"POST /api/v1/inpaint": { "POST /api/v1/inpaint": {
"count": 6264, "count": 26,
"avg_time": 0.5280383849143981, "avg_time": 1.5387938297711885,
"min_time": 0.2597086429595947, "min_time": 0.4340169429779053,
"max_time": 1.651228666305542, "max_time": 7.625817060470581,
"current_concurrent": 5 "current_concurrent": 2
}, },
"POST /api/v1/run_plugin_gen_image": { "GET /api/v1/realtime_status": {
"count": 463, "count": 20,
"avg_time": 0.4474348998069763, "avg_time": 0.002240335941314697,
"min_time": 0.1340315341949463, "min_time": 0.0006320476531982422,
"max_time": 2.5062549114227295, "max_time": 0.006146907806396484,
"current_concurrent": 1 "current_concurrent": 0
}, },
"POST /api/v1/remove_bg": { "POST /api/v1/remove_bg": {
"count": 0, "count": 0,
@ -117,14 +125,41 @@
"current_concurrent": 0 "current_concurrent": 0
} }
}, },
"average_response_time": 0.2757347109317779, "average_response_time": 0.524723462054604,
"min_response_time": 0.0005586147308349609, "min_response_time": 0.0006320476531982422,
"max_response_time": 2.8400533199310303, "max_response_time": 8.236981391906738,
"current_concurrent": 6, "current_concurrent": 2,
"max_concurrent": 12, "max_concurrent": 7,
"requests_per_second": 1.3742163667312552, "requests_per_second": 1.6502782030794676,
"uptime": 9799.76685333252, "uptime": 57.5660514831543,
"recent_errors": [] "recent_errors": []
}, },
"timestamp": 1759375540.6194317 "daily_stats": {
"date": "2025-10-02",
"images_processed": {
"inpaint": 26,
"remove_bg": 0,
"gen_image": 5,
"total": 31
},
"network": {
"bytes_uploaded": 39333149,
"bytes_downloaded": 25089494,
"requests_count": 95,
"mb_uploaded": 37.511013984680176,
"mb_downloaded": 23.92720603942871,
"gb_uploaded": 0.036631849594414234,
"gb_downloaded": 0.0233664121478796
},
"api_calls": {
"total": 95,
"success": 95,
"failed": 0
},
"models_used": {},
"peak_concurrent": 6,
"start_time": 1759384315.930352,
"last_update": 1759384373.4347339
},
"timestamp": 1759384373.5121431
} }