ckh08045
/
inpaintServer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

API 에러 로그 기능을 추가하고, 최근 API 에러를 대시보드에서 조회할 수 있는 기능을 구현하였습니다. 상태 JSON 파일의 워커 상태를 업데이트하고, 서버 로그에서 프로세스 ID 변경 사항을 반영하였습니다.

This commit is contained in:
vast 2025-10-01 08:18:21 +00:00
parent 01f96a2659
commit cebf7612c6
8 changed files with 454 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
app/monitoring/dashboard.py
View File

@ -413,6 +413,27 @@ class MonitoringData:
# 에러 로그 제한 # 에러 로그 제한
if len(self.api_stats["errors"]) > 50: if len(self.api_stats["errors"]) > 50:
self.api_stats["errors"] = self.api_stats["errors"][-50:] self.api_stats["errors"] = self.api_stats["errors"][-50:]
def read_recent_errors(self, limit: int = 50) -> List[Dict[str, Any]]:
"""메인 서버에서 기록한 logs/api_errors.jsonl 에서 최근 에러를 읽어옵니다."""
try:
log_path = os.path.join(settings.PROJECT_ROOT, "logs", "api_errors.jsonl")
if not os.path.exists(log_path):
return []
errors: List[Dict[str, Any]] = []
with open(log_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
errors.append(json.loads(line))
except Exception:
continue
return errors[-limit:]
except Exception as e:
logger.error(f"최근 에러 읽기 실패: {e}")
return []
def get_history(self) -> List[Dict[str, Any]]: def get_history(self) -> List[Dict[str, Any]]:
"""데이터 히스토리를 반환합니다.""" """데이터 히스토리를 반환합니다."""
@ -586,6 +607,24 @@ HTML_TEMPLATE = """
gap: 15px; gap: 15px;
margin-top: 15px; margin-top: 15px;
} }
.errors-container {
background: white;
border-radius: 8px;
padding: 15px;
border: 1px solid #e5e7eb;
}
.error-row {
display: grid;
grid-template-columns: 160px 80px 120px 1fr 80px;
gap: 8px;
padding: 8px 0;
border-bottom: 1px solid #f0f0f0;
font-family: 'Courier New', monospace;
font-size: 12px;
}
.error-row.header { font-weight: bold; background: #f8f9fa; }
.error-search { display: flex; gap: 8px; margin-bottom: 10px; }
.error-search input, .error-search select { padding: 6px 8px; }
.endpoint-item { .endpoint-item {
background: #f8f9fa; background: #f8f9fa;
padding: 15px; padding: 15px;
@ -945,6 +984,34 @@ HTML_TEMPLATE = """
</div> </div>
</div> </div>
<!-- 최근 에러 -->
<div class="card">
<h3>🚨 최근 API 에러</h3>
<div class="error-search">
<input type="text" id="error-endpoint-filter" placeholder="엔드포인트 포함 검색 (/api/...)" />
<select id="error-status-filter">
<option value="">상태코드 전체</option>
<option value="4">4xx만</option>
<option value="5">5xx만</option>
<option value="200">200</option>
<option value="400">400</option>
<option value="404">404</option>
<option value="500">500</option>
</select>
<button onclick="refreshErrors()" style="padding: 5px 12px;">새로고침</button>
</div>
<div class="errors-container" id="errors-container">
<div class="error-row header">
<div>시간</div>
<div>메서드</div>
<div>상태</div>
<div>엔드포인트</div>
<div>지연(ms)</div>
</div>
<div id="errors-body">데이터 없음</div>
</div>
</div>
<!-- 모델 처리 시간 통계 --> <!-- 모델 처리 시간 통계 -->
<div class="card"> <div class="card">
<h3>⏱️ 모델 처리 시간 통계 (초)</h3> <h3>⏱️ 모델 처리 시간 통계 (초)</h3>
@ -1323,6 +1390,47 @@ HTML_TEMPLATE = """
container.appendChild(item); container.appendChild(item);
}); });
} }
// 최근 에러 로딩/렌더링
function renderErrors(errors) {
const endpointFilter = (document.getElementById('error-endpoint-filter').value || '').toLowerCase();
const statusFilter = (document.getElementById('error-status-filter').value || '');
const container = document.getElementById('errors-body');
let filtered = errors || [];
if (endpointFilter) {
filtered = filtered.filter(e => (e.path || '').toLowerCase().includes(endpointFilter));
}
if (statusFilter) {
if (statusFilter === '4') filtered = filtered.filter(e => (e.status || 0) >= 400 && (e.status || 0) < 500);
else if (statusFilter === '5') filtered = filtered.filter(e => (e.status || 0) >= 500);
else filtered = filtered.filter(e => String(e.status || '') === statusFilter);
}
if (filtered.length === 0) {
container.innerHTML = '<div style="color:#999; padding:8px;">데이터 없음</div>';
return;
}
const rows = filtered.map(e => {
const ts = e.timestamp ? new Date((typeof e.timestamp === 'number' ? e.timestamp*1000 : Date.parse(e.timestamp))).toLocaleString() : '-';
const method = e.method || '-';
const path = e.path || '-';
const status = e.status != null ? e.status : '-';
const rt = e.response_time_ms != null ? e.response_time_ms : '-';
return `<div class="error-row"><div>${ts}</div><div>${method}</div><div>${status}</div><div>${path}</div><div>${rt}</div></div>`;
}).join('');
container.innerHTML = rows;
}
function refreshErrors() {
fetch('/api/errors')
.then(r => r.json())
.then(data => renderErrors(data.errors || []))
.catch(() => {
document.getElementById('errors-body').innerHTML = '<div style="color:#dc3545; padding:8px;">에러 로드 실패</div>';
});
}
function updateAlerts(alerts) { function updateAlerts(alerts) {
const container = document.getElementById('alerts-container'); const container = document.getElementById('alerts-container');
@ -1604,12 +1712,14 @@ HTML_TEMPLATE = """
refreshPerformanceStats(); refreshPerformanceStats();
refreshModelUsageStats(); refreshModelUsageStats();
refreshSystemAlerts(); refreshSystemAlerts();
refreshErrors();
// 30초마다 로그 및 성능 통계 자동 새로고침 // 30초마다 로그 및 성능 통계 자동 새로고침
setInterval(refreshLogs, 30000); setInterval(refreshLogs, 30000);
setInterval(refreshPerformanceStats, 30000); setInterval(refreshPerformanceStats, 30000);
setInterval(refreshModelUsageStats, 15000); // 15초마다 setInterval(refreshModelUsageStats, 15000); // 15초마다
setInterval(refreshSystemAlerts, 10000); // 10초마다 setInterval(refreshSystemAlerts, 10000); // 10초마다
setInterval(refreshErrors, 10000); // 10초마다
}); });
</script> </script>
</body> </body>
@ -1832,7 +1942,7 @@ async def get_system_alerts():
"""시스템 알림 반환""" """시스템 알림 반환"""
try: try:
alerts = [] alerts = []
# CPU 사용률 체크 # CPU 사용률 체크
cpu_percent = psutil.cpu_percent(interval=1) cpu_percent = psutil.cpu_percent(interval=1)
if cpu_percent > 90: if cpu_percent > 90:
@ -1844,19 +1954,19 @@ async def get_system_alerts():
}) })
elif cpu_percent > 75: elif cpu_percent > 75:
alerts.append({ alerts.append({
"level": "warning", "level": "warning",
"message": f"CPU 사용률이 높습니다: {cpu_percent:.1f}%", "message": f"CPU 사용률이 높습니다: {cpu_percent:.1f}%",
"category": "system", "category": "system",
"timestamp": datetime.now().isoformat() "timestamp": datetime.now().isoformat()
}) })
# 메모리 사용률 체크 # 메모리 사용률 체크
memory = psutil.virtual_memory() memory = psutil.virtual_memory()
if memory.percent > 90: if memory.percent > 90:
alerts.append({ alerts.append({
"level": "critical", "level": "critical",
"message": f"메모리 사용률이 매우 높습니다: {memory.percent:.1f}%", "message": f"메모리 사용률이 매우 높습니다: {memory.percent:.1f}%",
"category": "system", "category": "system",
"timestamp": datetime.now().isoformat() "timestamp": datetime.now().isoformat()
}) })
elif memory.percent > 80: elif memory.percent > 80:
@ -1866,13 +1976,13 @@ async def get_system_alerts():
"category": "system", "category": "system",
"timestamp": datetime.now().isoformat() "timestamp": datetime.now().isoformat()
}) })
# 디스크 사용률 체크 # 디스크 사용률 체크
disk = psutil.disk_usage('/') disk = psutil.disk_usage('/')
disk_percent = (disk.used / disk.total) * 100 disk_percent = (disk.used / disk.total) * 100
if disk_percent > 90: if disk_percent > 90:
alerts.append({ alerts.append({
"level": "critical", "level": "critical",
"message": f"디스크 사용률이 매우 높습니다: {disk_percent:.1f}%", "message": f"디스크 사용률이 매우 높습니다: {disk_percent:.1f}%",
"category": "system", "category": "system",
"timestamp": datetime.now().isoformat() "timestamp": datetime.now().isoformat()
@ -1880,11 +1990,11 @@ async def get_system_alerts():
elif disk_percent > 80: elif disk_percent > 80:
alerts.append({ alerts.append({
"level": "warning", "level": "warning",
"message": f"디스크 사용률이 높습니다: {disk_percent:.1f}%", "message": f"디스크 사용률이 높습니다: {disk_percent:.1f}%",
"category": "system", "category": "system",
"timestamp": datetime.now().isoformat() "timestamp": datetime.now().isoformat()
}) })
# GPU 메모리 체크 (가능한 경우) # GPU 메모리 체크 (가능한 경우)
try: try:
gpu_info = gpu_monitor.get_gpu_memory_info() gpu_info = gpu_monitor.get_gpu_memory_info()
@ -1897,13 +2007,22 @@ async def get_system_alerts():
}) })
except Exception: except Exception:
pass pass
return {"alerts": alerts, "count": len(alerts)} return {"alerts": alerts, "count": len(alerts)}
except Exception as e: except Exception as e:
logger.error(f"시스템 알림 조회 실패: {e}") logger.error(f"시스템 알림 조회 실패: {e}")
return {"alerts": [], "error": str(e)} return {"alerts": [], "error": str(e)}
@api_router.get("/errors", summary="최근 API 에러 목록")
def get_recent_errors(limit: int = 50):
"""최근 API 에러를 반환합니다 (logs/api_errors.jsonl 기반)."""
try:
return {"errors": monitoring_data.read_recent_errors(limit=limit)}
except Exception as e:
logger.error(f"에러 목록 조회 실패: {e}")
return {"errors": [], "error": str(e)}
@api_router.get("/test") @api_router.get("/test")
async def test_endpoint(): async def test_endpoint():
"""테스트용 엔드포인트입니다.""" """테스트용 엔드포인트입니다."""

215
logs/main.log
View File

@ -12682,3 +12682,218 @@
2025-10-01 07:26:14,897 - main - INFO - ✅ 배치 관리자 중지 완료 2025-10-01 07:26:14,897 - main - INFO - ✅ 배치 관리자 중지 완료
2025-10-01 07:26:14,897 - main - INFO - 👋 인페인팅 서버 종료 완료 2025-10-01 07:26:14,897 - main - INFO - 👋 인페인팅 서버 종료 완료
2025-10-01 07:26:14,897 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다. 2025-10-01 07:26:14,897 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 07:33:32,356 - main - INFO - 🚀 인페인팅 서버 시작 중...
2025-10-01 07:33:32,356 - main - INFO - ✅ 공유 객체를 app.state에 저장 완료
2025-10-01 07:33:32,356 - main - INFO - 🔄 상태 저장 백그라운드 작업 생성 중...
2025-10-01 07:33:32,356 - main - INFO - ✅ 상태 저장 백그라운드 작업 생성 완료
2025-10-01 07:33:32,356 - main - INFO - 🚀 세션 풀 초기화 (CUDA 자동 감지)
2025-10-01 07:33:32,356 - app.core.session_pool - INFO - Initializing dynamic session pools...
2025-10-01 07:33:32,356 - app.core.session_pool - INFO - Pre-loading 2 sessions for simple_lama
2025-10-01 07:33:32,356 - main - INFO - 🔄 상태 저장 백그라운드 작업 시작됨
2025-10-01 07:33:32,357 - app.core.session_pool - INFO - Creating new session simple_lama_0 for simple_lama...
2025-10-01 07:33:33,966 - app.core.session_pool - INFO - Creating new session simple_lama_1 for simple_lama...
2025-10-01 07:33:33,967 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 07:33:34,926 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 07:33:34,927 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 07:33:34,927 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 07:33:35,658 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 07:33:35,659 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 07:33:35,659 - app.core.session_pool - INFO - Successfully created session simple_lama_0
2025-10-01 07:33:35,660 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 07:33:35,660 - app.core.session_pool - INFO - Successfully created session simple_lama_1
2025-10-01 07:33:35,660 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 07:33:35,660 - app.core.session_pool - INFO - Pre-loading 1 sessions for migan
2025-10-01 07:33:35,660 - app.core.session_pool - INFO - Creating new session migan_0 for migan...
2025-10-01 07:33:35,698 - app.models.migan - INFO - Loading MIGAN ONNX model...
2025-10-01 07:33:35,698 - app.models.migan - INFO - MIGAN ONNX 런타임 세션 생성 시도...
2025-10-01 07:33:35,698 - app.models.migan - INFO - MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 07:33:35,969 - app.models.migan - INFO - MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 07:33:35,969 - app.models.migan - INFO - MIGAN ONNX model loaded successfully
2025-10-01 07:33:35,969 - app.core.session_pool - INFO - Successfully created session migan_0
2025-10-01 07:33:35,969 - app.core.session_pool - INFO - ➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%)
2025-10-01 07:33:35,970 - app.core.session_pool - INFO - Pre-loading 1 sessions for rembg
2025-10-01 07:33:35,970 - app.core.session_pool - INFO - Creating new session rembg_0 for rembg...
2025-10-01 07:33:35,971 - app.models.bria_rmbg_onnx - INFO - BriaRMBGOnnxProcessor 초기화 완료
2025-10-01 07:33:35,971 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx
2025-10-01 07:33:36,391 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output
2025-10-01 07:33:36,393 - app.core.session_pool - INFO - Successfully created session rembg_0
2025-10-01 07:33:36,393 - app.core.session_pool - INFO - ➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%)
2025-10-01 07:33:36,394 - app.core.session_pool - INFO - Session pools initialized successfully
2025-10-01 07:33:36,394 - main - INFO - ✅ 세션 풀 초기화 완료
2025-10-01 07:33:36,394 - app.core.worker_manager - INFO - Starting worker manager...
2025-10-01 07:33:36,394 - app.core.worker_manager - INFO - Worker manager started with 2 workers
2025-10-01 07:33:36,394 - main - INFO - ✅ 워커 매니저 시작 완료
2025-10-01 07:33:36,395 - app.core.batch_manager - INFO - Starting BatchManager...
2025-10-01 07:33:36,395 - app.core.batch_manager - INFO - BatchManager started successfully.
2025-10-01 07:33:36,395 - main - INFO - ✅ 배치 관리자 시작 완료
2025-10-01 07:33:36,395 - main - INFO - 🎉 인페인팅 서버 시작 완료!
2025-10-01 07:33:36,395 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 07:33:36,396 - app.core.session_pool - INFO - Idle session reaper started. Timeout: 1800s, Check Interval: 60s
2025-10-01 07:33:58,305 - main - INFO - 🛑 인페인팅 서버 종료 중...
2025-10-01 07:33:58,306 - app.core.worker_manager - INFO - Stopping worker manager...
2025-10-01 07:33:58,306 - app.core.worker_manager - INFO - Worker manager stopped
2025-10-01 07:33:58,307 - main - INFO - ✅ 워커 매니저 중지 완료
2025-10-01 07:33:58,307 - app.core.batch_manager - INFO - Stopping BatchManager...
2025-10-01 07:33:58,307 - app.core.batch_manager - INFO - BatchManager stopped.
2025-10-01 07:33:58,308 - main - INFO - ✅ 배치 관리자 중지 완료
2025-10-01 07:33:58,308 - main - INFO - 👋 인페인팅 서버 종료 완료
2025-10-01 07:33:58,308 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 07:48:56,057 - main - INFO - 🚀 인페인팅 서버 시작 중...
2025-10-01 07:48:56,057 - main - INFO - ✅ 공유 객체를 app.state에 저장 완료
2025-10-01 07:48:56,057 - main - INFO - 🔄 상태 저장 백그라운드 작업 생성 중...
2025-10-01 07:48:56,057 - main - INFO - ✅ 상태 저장 백그라운드 작업 생성 완료
2025-10-01 07:48:56,057 - main - INFO - 🚀 세션 풀 초기화 (CUDA 자동 감지)
2025-10-01 07:48:56,057 - app.core.session_pool - INFO - Initializing dynamic session pools...
2025-10-01 07:48:56,057 - app.core.session_pool - INFO - Pre-loading 2 sessions for simple_lama
2025-10-01 07:48:56,057 - main - INFO - 🔄 상태 저장 백그라운드 작업 시작됨
2025-10-01 07:48:56,058 - app.core.session_pool - INFO - Creating new session simple_lama_0 for simple_lama...
2025-10-01 07:48:57,657 - app.core.session_pool - INFO - Creating new session simple_lama_1 for simple_lama...
2025-10-01 07:48:57,657 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 07:48:58,608 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 07:48:58,608 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 07:48:58,609 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 07:48:59,372 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 07:48:59,373 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 07:48:59,373 - app.core.session_pool - INFO - Successfully created session simple_lama_0
2025-10-01 07:48:59,373 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 07:48:59,374 - app.core.session_pool - INFO - Successfully created session simple_lama_1
2025-10-01 07:48:59,374 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 07:48:59,374 - app.core.session_pool - INFO - Pre-loading 1 sessions for migan
2025-10-01 07:48:59,374 - app.core.session_pool - INFO - Creating new session migan_0 for migan...
2025-10-01 07:48:59,414 - app.models.migan - INFO - Loading MIGAN ONNX model...
2025-10-01 07:48:59,414 - app.models.migan - INFO - MIGAN ONNX 런타임 세션 생성 시도...
2025-10-01 07:48:59,414 - app.models.migan - INFO - MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 07:48:59,681 - app.models.migan - INFO - MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 07:48:59,682 - app.models.migan - INFO - MIGAN ONNX model loaded successfully
2025-10-01 07:48:59,682 - app.core.session_pool - INFO - Successfully created session migan_0
2025-10-01 07:48:59,682 - app.core.session_pool - INFO - ➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%)
2025-10-01 07:48:59,682 - app.core.session_pool - INFO - Pre-loading 1 sessions for rembg
2025-10-01 07:48:59,682 - app.core.session_pool - INFO - Creating new session rembg_0 for rembg...
2025-10-01 07:48:59,683 - app.models.bria_rmbg_onnx - INFO - BriaRMBGOnnxProcessor 초기화 완료
2025-10-01 07:48:59,684 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx
2025-10-01 07:49:00,102 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output
2025-10-01 07:49:00,103 - app.core.session_pool - INFO - Successfully created session rembg_0
2025-10-01 07:49:00,104 - app.core.session_pool - INFO - ➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%)
2025-10-01 07:49:00,104 - app.core.session_pool - INFO - Session pools initialized successfully
2025-10-01 07:49:00,104 - main - INFO - ✅ 세션 풀 초기화 완료
2025-10-01 07:49:00,105 - app.core.worker_manager - INFO - Starting worker manager...
2025-10-01 07:49:00,105 - app.core.worker_manager - INFO - Worker manager started with 2 workers
2025-10-01 07:49:00,105 - main - INFO - ✅ 워커 매니저 시작 완료
2025-10-01 07:49:00,105 - app.core.batch_manager - INFO - Starting BatchManager...
2025-10-01 07:49:00,105 - app.core.batch_manager - INFO - BatchManager started successfully.
2025-10-01 07:49:00,105 - main - INFO - ✅ 배치 관리자 시작 완료
2025-10-01 07:49:00,106 - main - INFO - 🎉 인페인팅 서버 시작 완료!
2025-10-01 07:49:00,106 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 07:49:00,107 - app.core.session_pool - INFO - Idle session reaper started. Timeout: 1800s, Check Interval: 60s
2025-10-01 08:03:59,983 - main - INFO - 🛑 인페인팅 서버 종료 중...
2025-10-01 08:03:59,984 - app.core.worker_manager - INFO - Stopping worker manager...
2025-10-01 08:03:59,984 - app.core.worker_manager - INFO - Worker manager stopped
2025-10-01 08:03:59,985 - main - INFO - ✅ 워커 매니저 중지 완료
2025-10-01 08:03:59,985 - app.core.batch_manager - INFO - Stopping BatchManager...
2025-10-01 08:03:59,985 - app.core.batch_manager - INFO - BatchManager stopped.
2025-10-01 08:03:59,986 - main - INFO - ✅ 배치 관리자 중지 완료
2025-10-01 08:03:59,986 - main - INFO - 👋 인페인팅 서버 종료 완료
2025-10-01 08:03:59,986 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 08:04:21,165 - main - INFO - 🚀 인페인팅 서버 시작 중...
2025-10-01 08:04:21,165 - main - INFO - ✅ 공유 객체를 app.state에 저장 완료
2025-10-01 08:04:21,165 - main - INFO - 🔄 상태 저장 백그라운드 작업 생성 중...
2025-10-01 08:04:21,166 - main - INFO - ✅ 상태 저장 백그라운드 작업 생성 완료
2025-10-01 08:04:21,166 - main - INFO - 🚀 세션 풀 초기화 (CUDA 자동 감지)
2025-10-01 08:04:21,166 - app.core.session_pool - INFO - Initializing dynamic session pools...
2025-10-01 08:04:21,166 - app.core.session_pool - INFO - Pre-loading 2 sessions for simple_lama
2025-10-01 08:04:21,166 - main - INFO - 🔄 상태 저장 백그라운드 작업 시작됨
2025-10-01 08:04:21,167 - app.core.session_pool - INFO - Creating new session simple_lama_0 for simple_lama...
2025-10-01 08:04:22,780 - app.core.session_pool - INFO - Creating new session simple_lama_1 for simple_lama...
2025-10-01 08:04:22,781 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 08:04:23,735 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 08:04:23,735 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 08:04:23,736 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 08:04:24,458 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 08:04:24,459 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 08:04:24,459 - app.core.session_pool - INFO - Successfully created session simple_lama_0
2025-10-01 08:04:24,460 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 08:04:24,460 - app.core.session_pool - INFO - Successfully created session simple_lama_1
2025-10-01 08:04:24,460 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 08:04:24,460 - app.core.session_pool - INFO - Pre-loading 1 sessions for migan
2025-10-01 08:04:24,460 - app.core.session_pool - INFO - Creating new session migan_0 for migan...
2025-10-01 08:04:24,498 - app.models.migan - INFO - Loading MIGAN ONNX model...
2025-10-01 08:04:24,498 - app.models.migan - INFO - MIGAN ONNX 런타임 세션 생성 시도...
2025-10-01 08:04:24,498 - app.models.migan - INFO - MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 08:04:24,773 - app.models.migan - INFO - MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 08:04:24,774 - app.models.migan - INFO - MIGAN ONNX model loaded successfully
2025-10-01 08:04:24,774 - app.core.session_pool - INFO - Successfully created session migan_0
2025-10-01 08:04:24,774 - app.core.session_pool - INFO - ➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%)
2025-10-01 08:04:24,774 - app.core.session_pool - INFO - Pre-loading 1 sessions for rembg
2025-10-01 08:04:24,774 - app.core.session_pool - INFO - Creating new session rembg_0 for rembg...
2025-10-01 08:04:24,775 - app.models.bria_rmbg_onnx - INFO - BriaRMBGOnnxProcessor 초기화 완료
2025-10-01 08:04:24,775 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx
2025-10-01 08:04:25,167 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output
2025-10-01 08:04:25,169 - app.core.session_pool - INFO - Successfully created session rembg_0
2025-10-01 08:04:25,169 - app.core.session_pool - INFO - ➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%)
2025-10-01 08:04:25,170 - app.core.session_pool - INFO - Session pools initialized successfully
2025-10-01 08:04:25,170 - main - INFO - ✅ 세션 풀 초기화 완료
2025-10-01 08:04:25,170 - app.core.worker_manager - INFO - Starting worker manager...
2025-10-01 08:04:25,170 - app.core.worker_manager - INFO - Worker manager started with 2 workers
2025-10-01 08:04:25,170 - main - INFO - ✅ 워커 매니저 시작 완료
2025-10-01 08:04:25,171 - app.core.batch_manager - INFO - Starting BatchManager...
2025-10-01 08:04:25,171 - app.core.batch_manager - INFO - BatchManager started successfully.
2025-10-01 08:04:25,171 - main - INFO - ✅ 배치 관리자 시작 완료
2025-10-01 08:04:25,171 - main - INFO - 🎉 인페인팅 서버 시작 완료!
2025-10-01 08:04:25,172 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 08:04:25,172 - app.core.session_pool - INFO - Idle session reaper started. Timeout: 1800s, Check Interval: 60s
2025-10-01 08:17:55,918 - main - INFO - 🛑 인페인팅 서버 종료 중...
2025-10-01 08:17:55,918 - app.core.worker_manager - INFO - Stopping worker manager...
2025-10-01 08:17:55,919 - app.core.worker_manager - INFO - Worker manager stopped
2025-10-01 08:17:55,919 - main - INFO - ✅ 워커 매니저 중지 완료
2025-10-01 08:17:55,919 - app.core.batch_manager - INFO - Stopping BatchManager...
2025-10-01 08:17:55,920 - app.core.batch_manager - INFO - BatchManager stopped.
2025-10-01 08:17:55,920 - main - INFO - ✅ 배치 관리자 중지 완료
2025-10-01 08:17:55,920 - main - INFO - 👋 인페인팅 서버 종료 완료
2025-10-01 08:17:55,921 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 08:18:14,049 - main - INFO - 🚀 인페인팅 서버 시작 중...
2025-10-01 08:18:14,050 - main - INFO - ✅ 공유 객체를 app.state에 저장 완료
2025-10-01 08:18:14,050 - main - INFO - 🔄 상태 저장 백그라운드 작업 생성 중...
2025-10-01 08:18:14,050 - main - INFO - ✅ 상태 저장 백그라운드 작업 생성 완료
2025-10-01 08:18:14,050 - main - INFO - 🚀 세션 풀 초기화 (CUDA 자동 감지)
2025-10-01 08:18:14,050 - app.core.session_pool - INFO - Initializing dynamic session pools...
2025-10-01 08:18:14,050 - app.core.session_pool - INFO - Pre-loading 2 sessions for simple_lama
2025-10-01 08:18:14,050 - main - INFO - 🔄 상태 저장 백그라운드 작업 시작됨
2025-10-01 08:18:14,051 - app.core.session_pool - INFO - Creating new session simple_lama_0 for simple_lama...
2025-10-01 08:18:15,702 - app.core.session_pool - INFO - Creating new session simple_lama_1 for simple_lama...
2025-10-01 08:18:15,702 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 08:18:16,693 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 08:18:16,694 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 08:18:16,694 - app.models.simple_lama - INFO - Loading Simple LAMA model...
2025-10-01 08:18:17,473 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
2025-10-01 08:18:17,473 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
2025-10-01 08:18:17,473 - app.core.session_pool - INFO - Successfully created session simple_lama_0
2025-10-01 08:18:17,474 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 08:18:17,474 - app.core.session_pool - INFO - Successfully created session simple_lama_1
2025-10-01 08:18:17,474 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 08:18:17,474 - app.core.session_pool - INFO - Pre-loading 1 sessions for migan
2025-10-01 08:18:17,475 - app.core.session_pool - INFO - Creating new session migan_0 for migan...
2025-10-01 08:18:17,514 - app.models.migan - INFO - Loading MIGAN ONNX model...
2025-10-01 08:18:17,514 - app.models.migan - INFO - MIGAN ONNX 런타임 세션 생성 시도...
2025-10-01 08:18:17,514 - app.models.migan - INFO - MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 08:18:17,785 - app.models.migan - INFO - MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 08:18:17,785 - app.models.migan - INFO - MIGAN ONNX model loaded successfully
2025-10-01 08:18:17,785 - app.core.session_pool - INFO - Successfully created session migan_0
2025-10-01 08:18:17,786 - app.core.session_pool - INFO - ➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%)
2025-10-01 08:18:17,786 - app.core.session_pool - INFO - Pre-loading 1 sessions for rembg
2025-10-01 08:18:17,786 - app.core.session_pool - INFO - Creating new session rembg_0 for rembg...
2025-10-01 08:18:17,787 - app.models.bria_rmbg_onnx - INFO - BriaRMBGOnnxProcessor 초기화 완료
2025-10-01 08:18:17,787 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx
2025-10-01 08:18:18,181 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output
2025-10-01 08:18:18,183 - app.core.session_pool - INFO - Successfully created session rembg_0
2025-10-01 08:18:18,183 - app.core.session_pool - INFO - ➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%)
2025-10-01 08:18:18,184 - app.core.session_pool - INFO - Session pools initialized successfully
2025-10-01 08:18:18,184 - main - INFO - ✅ 세션 풀 초기화 완료
2025-10-01 08:18:18,184 - app.core.worker_manager - INFO - Starting worker manager...
2025-10-01 08:18:18,185 - app.core.worker_manager - INFO - Worker manager started with 2 workers
2025-10-01 08:18:18,185 - main - INFO - ✅ 워커 매니저 시작 완료
2025-10-01 08:18:18,185 - app.core.batch_manager - INFO - Starting BatchManager...
2025-10-01 08:18:18,185 - app.core.batch_manager - INFO - BatchManager started successfully.
2025-10-01 08:18:18,185 - main - INFO - ✅ 배치 관리자 시작 완료
2025-10-01 08:18:18,185 - main - INFO - 🎉 인페인팅 서버 시작 완료!
2025-10-01 08:18:18,186 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 08:18:18,186 - app.core.session_pool - INFO - Idle session reaper started. Timeout: 1800s, Check Interval: 60s

152
logs/main_server.log
View File

@ -1,152 +1,102 @@
WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats' WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats'
INFO: Started server process [181051] INFO: Started server process [192534]
INFO: Waiting for application startup. INFO: Waiting for application startup.
INFO:main:🚀 인페인팅 서버 시작 중... INFO:main:🚀 인페인팅 서버 시작 중...
2025-10-01 07:25:19,104 - main - INFO - 🚀 인페인팅 서버 시작 중... 2025-10-01 08:18:14,049 - main - INFO - 🚀 인페인팅 서버 시작 중...
INFO:main:✅ 공유 객체를 app.state에 저장 완료 INFO:main:✅ 공유 객체를 app.state에 저장 완료
2025-10-01 07:25:19,104 - main - INFO - ✅ 공유 객체를 app.state에 저장 완료 2025-10-01 08:18:14,050 - main - INFO - ✅ 공유 객체를 app.state에 저장 완료
INFO:main:🔄 상태 저장 백그라운드 작업 생성 중... INFO:main:🔄 상태 저장 백그라운드 작업 생성 중...
2025-10-01 07:25:19,104 - main - INFO - 🔄 상태 저장 백그라운드 작업 생성 중... 2025-10-01 08:18:14,050 - main - INFO - 🔄 상태 저장 백그라운드 작업 생성 중...
INFO:main:✅ 상태 저장 백그라운드 작업 생성 완료 INFO:main:✅ 상태 저장 백그라운드 작업 생성 완료
2025-10-01 07:25:19,105 - main - INFO - ✅ 상태 저장 백그라운드 작업 생성 완료 2025-10-01 08:18:14,050 - main - INFO - ✅ 상태 저장 백그라운드 작업 생성 완료
INFO:main:🚀 세션 풀 초기화 (CUDA 자동 감지) INFO:main:🚀 세션 풀 초기화 (CUDA 자동 감지)
2025-10-01 07:25:19,105 - main - INFO - 🚀 세션 풀 초기화 (CUDA 자동 감지) 2025-10-01 08:18:14,050 - main - INFO - 🚀 세션 풀 초기화 (CUDA 자동 감지)
INFO:app.core.session_pool:Initializing dynamic session pools... INFO:app.core.session_pool:Initializing dynamic session pools...
2025-10-01 07:25:19,105 - app.core.session_pool - INFO - Initializing dynamic session pools... 2025-10-01 08:18:14,050 - app.core.session_pool - INFO - Initializing dynamic session pools...
INFO:app.core.session_pool:Pre-loading 2 sessions for simple_lama INFO:app.core.session_pool:Pre-loading 2 sessions for simple_lama
2025-10-01 07:25:19,105 - app.core.session_pool - INFO - Pre-loading 2 sessions for simple_lama 2025-10-01 08:18:14,050 - app.core.session_pool - INFO - Pre-loading 2 sessions for simple_lama
INFO:main:🔄 상태 저장 백그라운드 작업 시작됨 INFO:main:🔄 상태 저장 백그라운드 작업 시작됨
2025-10-01 07:25:19,105 - main - INFO - 🔄 상태 저장 백그라운드 작업 시작됨 2025-10-01 08:18:14,050 - main - INFO - 🔄 상태 저장 백그라운드 작업 시작됨
INFO:app.core.session_pool:Creating new session simple_lama_0 for simple_lama... INFO:app.core.session_pool:Creating new session simple_lama_0 for simple_lama...
2025-10-01 07:25:19,106 - app.core.session_pool - INFO - Creating new session simple_lama_0 for simple_lama... 2025-10-01 08:18:14,051 - app.core.session_pool - INFO - Creating new session simple_lama_0 for simple_lama...
INFO:app.core.session_pool:Creating new session simple_lama_1 for simple_lama... INFO:app.core.session_pool:Creating new session simple_lama_1 for simple_lama...
2025-10-01 07:25:20,756 - app.core.session_pool - INFO - Creating new session simple_lama_1 for simple_lama... 2025-10-01 08:18:15,702 - app.core.session_pool - INFO - Creating new session simple_lama_1 for simple_lama...
INFO:app.models.simple_lama:Loading Simple LAMA model... INFO:app.models.simple_lama:Loading Simple LAMA model...
2025-10-01 07:25:20,756 - app.models.simple_lama - INFO - Loading Simple LAMA model... 2025-10-01 08:18:15,702 - app.models.simple_lama - INFO - Loading Simple LAMA model...
INFO:app.models.simple_lama:실제 SimpleLama 모델 로딩 완료 INFO:app.models.simple_lama:실제 SimpleLama 모델 로딩 완료
2025-10-01 07:25:21,717 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료 2025-10-01 08:18:16,693 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
INFO:app.models.simple_lama:Simple LAMA model loaded successfully INFO:app.models.simple_lama:Simple LAMA model loaded successfully
2025-10-01 07:25:21,718 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully 2025-10-01 08:18:16,694 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
INFO:app.models.simple_lama:Loading Simple LAMA model... INFO:app.models.simple_lama:Loading Simple LAMA model...
2025-10-01 07:25:21,718 - app.models.simple_lama - INFO - Loading Simple LAMA model... 2025-10-01 08:18:16,694 - app.models.simple_lama - INFO - Loading Simple LAMA model...
INFO:app.models.simple_lama:실제 SimpleLama 모델 로딩 완료 INFO:app.models.simple_lama:실제 SimpleLama 모델 로딩 완료
2025-10-01 07:25:22,473 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료 2025-10-01 08:18:17,473 - app.models.simple_lama - INFO - 실제 SimpleLama 모델 로딩 완료
INFO:app.models.simple_lama:Simple LAMA model loaded successfully INFO:app.models.simple_lama:Simple LAMA model loaded successfully
2025-10-01 07:25:22,473 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully 2025-10-01 08:18:17,473 - app.models.simple_lama - INFO - Simple LAMA model loaded successfully
INFO:app.core.session_pool:Successfully created session simple_lama_0 INFO:app.core.session_pool:Successfully created session simple_lama_0
2025-10-01 07:25:22,473 - app.core.session_pool - INFO - Successfully created session simple_lama_0 2025-10-01 08:18:17,473 - app.core.session_pool - INFO - Successfully created session simple_lama_0
INFO:app.core.session_pool:➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%) INFO:app.core.session_pool:➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 07:25:22,474 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%) 2025-10-01 08:18:17,474 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
INFO:app.core.session_pool:Successfully created session simple_lama_1 INFO:app.core.session_pool:Successfully created session simple_lama_1
2025-10-01 07:25:22,474 - app.core.session_pool - INFO - Successfully created session simple_lama_1 2025-10-01 08:18:17,474 - app.core.session_pool - INFO - Successfully created session simple_lama_1
INFO:app.core.session_pool:➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%) INFO:app.core.session_pool:➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
2025-10-01 07:25:22,474 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%) 2025-10-01 08:18:17,474 - app.core.session_pool - INFO - ➕ Session Created (simple_lama). Status -> simple_lama: 0, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.3%)
INFO:app.core.session_pool:Pre-loading 1 sessions for migan INFO:app.core.session_pool:Pre-loading 1 sessions for migan
2025-10-01 07:25:22,474 - app.core.session_pool - INFO - Pre-loading 1 sessions for migan 2025-10-01 08:18:17,474 - app.core.session_pool - INFO - Pre-loading 1 sessions for migan
INFO:app.core.session_pool:Creating new session migan_0 for migan... INFO:app.core.session_pool:Creating new session migan_0 for migan...
2025-10-01 07:25:22,474 - app.core.session_pool - INFO - Creating new session migan_0 for migan... 2025-10-01 08:18:17,475 - app.core.session_pool - INFO - Creating new session migan_0 for migan...
INFO:app.models.migan:Loading MIGAN ONNX model... INFO:app.models.migan:Loading MIGAN ONNX model...
2025-10-01 07:25:22,514 - app.models.migan - INFO - Loading MIGAN ONNX model... 2025-10-01 08:18:17,514 - app.models.migan - INFO - Loading MIGAN ONNX model...
INFO:app.models.migan:MIGAN ONNX 런타임 세션 생성 시도... INFO:app.models.migan:MIGAN ONNX 런타임 세션 생성 시도...
2025-10-01 07:25:22,514 - app.models.migan - INFO - MIGAN ONNX 런타임 세션 생성 시도... 2025-10-01 08:18:17,514 - app.models.migan - INFO - MIGAN ONNX 런타임 세션 생성 시도...
INFO:app.models.migan:MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider'] INFO:app.models.migan:MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 07:25:22,514 - app.models.migan - INFO - MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider'] 2025-10-01 08:18:17,514 - app.models.migan - INFO - MIGAN ONNX providers 설정: ['CUDAExecutionProvider', 'CPUExecutionProvider']
INFO:app.models.migan:MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] INFO:app.models.migan:MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
2025-10-01 07:25:22,782 - app.models.migan - INFO - MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] 2025-10-01 08:18:17,785 - app.models.migan - INFO - MIGAN ONNX 세션 생성 완료. Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']
INFO:app.models.migan:MIGAN ONNX model loaded successfully INFO:app.models.migan:MIGAN ONNX model loaded successfully
2025-10-01 07:25:22,782 - app.models.migan - INFO - MIGAN ONNX model loaded successfully 2025-10-01 08:18:17,785 - app.models.migan - INFO - MIGAN ONNX model loaded successfully
INFO:app.core.session_pool:Successfully created session migan_0 INFO:app.core.session_pool:Successfully created session migan_0
2025-10-01 07:25:22,782 - app.core.session_pool - INFO - Successfully created session migan_0 2025-10-01 08:18:17,785 - app.core.session_pool - INFO - Successfully created session migan_0
INFO:app.core.session_pool:➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%) INFO:app.core.session_pool:➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%)
2025-10-01 07:25:22,782 - app.core.session_pool - INFO - ➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%) 2025-10-01 08:18:17,786 - app.core.session_pool - INFO - ➕ Session Created (migan). Status -> simple_lama: 2, migan: 0, rembg: 0 | VRAM: 0.0/0.0 GB (7.7%)
INFO:app.core.session_pool:Pre-loading 1 sessions for rembg INFO:app.core.session_pool:Pre-loading 1 sessions for rembg
2025-10-01 07:25:22,783 - app.core.session_pool - INFO - Pre-loading 1 sessions for rembg 2025-10-01 08:18:17,786 - app.core.session_pool - INFO - Pre-loading 1 sessions for rembg
INFO:app.core.session_pool:Creating new session rembg_0 for rembg... INFO:app.core.session_pool:Creating new session rembg_0 for rembg...
2025-10-01 07:25:22,783 - app.core.session_pool - INFO - Creating new session rembg_0 for rembg... 2025-10-01 08:18:17,786 - app.core.session_pool - INFO - Creating new session rembg_0 for rembg...
INFO:app.models.bria_rmbg_onnx:BriaRMBGOnnxProcessor 초기화 완료 INFO:app.models.bria_rmbg_onnx:BriaRMBGOnnxProcessor 초기화 완료
2025-10-01 07:25:22,784 - app.models.bria_rmbg_onnx - INFO - BriaRMBGOnnxProcessor 초기화 완료 2025-10-01 08:18:17,787 - app.models.bria_rmbg_onnx - INFO - BriaRMBGOnnxProcessor 초기화 완료
INFO:app.models.bria_rmbg_onnx:Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx INFO:app.models.bria_rmbg_onnx:Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx
2025-10-01 07:25:22,784 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx 2025-10-01 08:18:17,787 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 중... path=app/models/onnx/BriaRMBG1.4_model_fp16.onnx
INFO:app.models.bria_rmbg_onnx:Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output INFO:app.models.bria_rmbg_onnx:Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output
2025-10-01 07:25:23,185 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output 2025-10-01 08:18:18,181 - app.models.bria_rmbg_onnx - INFO - Bria RMBG ONNX 세션 생성 완료, Providers: ['CUDAExecutionProvider', 'CPUExecutionProvider'] | Input: input, Output: output
INFO:app.core.session_pool:Successfully created session rembg_0 INFO:app.core.session_pool:Successfully created session rembg_0
2025-10-01 07:25:23,186 - app.core.session_pool - INFO - Successfully created session rembg_0 2025-10-01 08:18:18,183 - app.core.session_pool - INFO - Successfully created session rembg_0
INFO:app.core.session_pool:➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%) INFO:app.core.session_pool:➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%)
2025-10-01 07:25:23,187 - app.core.session_pool - INFO - ➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%) 2025-10-01 08:18:18,183 - app.core.session_pool - INFO - ➕ Session Created (rembg). Status -> simple_lama: 2, migan: 1, rembg: 0 | VRAM: 0.0/0.0 GB (8.9%)
INFO:app.core.session_pool:Session pools initialized successfully INFO:app.core.session_pool:Session pools initialized successfully
2025-10-01 07:25:23,187 - app.core.session_pool - INFO - Session pools initialized successfully 2025-10-01 08:18:18,184 - app.core.session_pool - INFO - Session pools initialized successfully
INFO:main:✅ 세션 풀 초기화 완료 INFO:main:✅ 세션 풀 초기화 완료
2025-10-01 07:25:23,188 - main - INFO - ✅ 세션 풀 초기화 완료 2025-10-01 08:18:18,184 - main - INFO - ✅ 세션 풀 초기화 완료
INFO:app.core.worker_manager:Starting worker manager... INFO:app.core.worker_manager:Starting worker manager...
2025-10-01 07:25:23,188 - app.core.worker_manager - INFO - Starting worker manager... 2025-10-01 08:18:18,184 - app.core.worker_manager - INFO - Starting worker manager...
INFO:app.core.worker_manager:Worker manager started with 2 workers INFO:app.core.worker_manager:Worker manager started with 2 workers
2025-10-01 07:25:23,188 - app.core.worker_manager - INFO - Worker manager started with 2 workers 2025-10-01 08:18:18,185 - app.core.worker_manager - INFO - Worker manager started with 2 workers
INFO:main:✅ 워커 매니저 시작 완료 INFO:main:✅ 워커 매니저 시작 완료
2025-10-01 07:25:23,188 - main - INFO - ✅ 워커 매니저 시작 완료 2025-10-01 08:18:18,185 - main - INFO - ✅ 워커 매니저 시작 완료
INFO:app.core.batch_manager:Starting BatchManager... INFO:app.core.batch_manager:Starting BatchManager...
2025-10-01 07:25:23,188 - app.core.batch_manager - INFO - Starting BatchManager... 2025-10-01 08:18:18,185 - app.core.batch_manager - INFO - Starting BatchManager...
INFO:app.core.batch_manager:BatchManager started successfully. INFO:app.core.batch_manager:BatchManager started successfully.
2025-10-01 07:25:23,189 - app.core.batch_manager - INFO - BatchManager started successfully. 2025-10-01 08:18:18,185 - app.core.batch_manager - INFO - BatchManager started successfully.
INFO:main:✅ 배치 관리자 시작 완료 INFO:main:✅ 배치 관리자 시작 완료
2025-10-01 07:25:23,189 - main - INFO - ✅ 배치 관리자 시작 완료 2025-10-01 08:18:18,185 - main - INFO - ✅ 배치 관리자 시작 완료
INFO:main:🎉 인페인팅 서버 시작 완료! INFO:main:🎉 인페인팅 서버 시작 완료!
2025-10-01 07:25:23,189 - main - INFO - 🎉 인페인팅 서버 시작 완료! 2025-10-01 08:18:18,185 - main - INFO - 🎉 인페인팅 서버 시작 완료!
WARNING:app.utils.discord_notifier:Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다. WARNING:app.utils.discord_notifier:Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 07:25:23,189 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다. 2025-10-01 08:18:18,186 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
INFO:app.core.session_pool:Idle session reaper started. Timeout: 1800s, Check Interval: 60s INFO:app.core.session_pool:Idle session reaper started. Timeout: 1800s, Check Interval: 60s
2025-10-01 07:25:23,190 - app.core.session_pool - INFO - Idle session reaper started. Timeout: 1800s, Check Interval: 60s 2025-10-01 08:18:18,186 - app.core.session_pool - INFO - Idle session reaper started. Timeout: 1800s, Check Interval: 60s
INFO: Application startup complete. INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:8008 (Press CTRL+C to quit) INFO: Uvicorn running on http://0.0.0.0:8008 (Press CTRL+C to quit)
INFO: 127.0.0.1:39514 - "GET /api/v1/health HTTP/1.1" 200 OK INFO: 127.0.0.1:57608 - "GET /api/v1/health HTTP/1.1" 200 OK
INFO: 127.0.0.1:39524 - "GET /api/v1/health HTTP/1.1" 200 OK INFO: 127.0.0.1:57612 - "GET /api/v1/health HTTP/1.1" 200 OK
INFO: 127.0.0.1:39532 - "GET /api/v1/stats HTTP/1.1" 200 OK INFO: 127.0.0.1:57626 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:39546 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40382 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40396 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40412 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40416 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40418 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40434 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:40438 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:54526 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:54532 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:54542 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:54554 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:54568 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:54574 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:47740 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:47752 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:47764 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:47778 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:47788 - "GET /api/v1/health HTTP/1.1" 200 OK
INFO: 127.0.0.1:47798 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:47808 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:33910 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:33922 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:33936 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:33952 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:33960 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:33968 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:44688 - "GET /api/v1/stats HTTP/1.1" 200 OK
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO:main:🛑 인페인팅 서버 종료 중...
2025-10-01 07:26:14,895 - main - INFO - 🛑 인페인팅 서버 종료 중...
INFO:app.core.worker_manager:Stopping worker manager...
2025-10-01 07:26:14,895 - app.core.worker_manager - INFO - Stopping worker manager...
INFO:app.core.worker_manager:Worker manager stopped
2025-10-01 07:26:14,896 - app.core.worker_manager - INFO - Worker manager stopped
INFO:main:✅ 워커 매니저 중지 완료
2025-10-01 07:26:14,896 - main - INFO - ✅ 워커 매니저 중지 완료
INFO:app.core.batch_manager:Stopping BatchManager...
2025-10-01 07:26:14,896 - app.core.batch_manager - INFO - Stopping BatchManager...
INFO:app.core.batch_manager:BatchManager stopped.
2025-10-01 07:26:14,897 - app.core.batch_manager - INFO - BatchManager stopped.
INFO:main:✅ 배치 관리자 중지 완료
2025-10-01 07:26:14,897 - main - INFO - ✅ 배치 관리자 중지 완료
INFO:main:👋 인페인팅 서버 종료 완료
2025-10-01 07:26:14,897 - main - INFO - 👋 인페인팅 서버 종료 완료
WARNING:app.utils.discord_notifier:Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
2025-10-01 07:26:14,897 - app.utils.discord_notifier - WARNING - Discord 웹훅 URL이 설정되지 않아 알림을 보낼 수 없습니다.
INFO: Application shutdown complete.
INFO: Finished server process [181051]

2
logs/main_server.pid
View File

@ -1 +1 @@
181051 192534

31
logs/monitoring.log
View File

@ -1,34 +1,7 @@
WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats' WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats'
INFO: Started server process [181223] INFO: Started server process [192741]
INFO: Waiting for application startup. INFO: Waiting for application startup.
INFO: Application startup complete. INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:8888 (Press CTRL+C to quit) INFO: Uvicorn running on http://0.0.0.0:8888 (Press CTRL+C to quit)
INFO: 122.35.47.45:60427 - "GET / HTTP/1.1" 200 OK INFO: 122.35.47.45:63439 - "WebSocket /ws" [accepted]
INFO: 122.35.47.45:60430 - "WebSocket /ws" [accepted]
INFO: connection open INFO: connection open
INFO: 122.35.47.45:60427 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:60428 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:60433 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:60434 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 127.0.0.1:42786 - "GET /api/simple HTTP/1.1" 200 OK
INFO: 122.35.47.45:60441 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:60441 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:60441 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:60460 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:60457 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:60458 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:60459 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:60472 - "GET /api/system-alerts HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Max retries exceeded with url: /api/v1/stats (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7feae88ed240>: Failed to establish a new connection: [Errno 111] Connection refused'))
INFO: 122.35.47.45:60472 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Max retries exceeded with url: /api/v1/stats (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7feae88ec2e0>: Failed to establish a new connection: [Errno 111] Connection refused'))
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Max retries exceeded with url: /api/v1/stats (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7feae88ed9f0>: Failed to establish a new connection: [Errno 111] Connection refused'))
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Max retries exceeded with url: /api/v1/stats (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7feae88ecbe0>: Failed to establish a new connection: [Errno 111] Connection refused'))
INFO: Shutting down
INFO: connection closed
INFO: Waiting for background tasks to complete. (CTRL+C to force quit)
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Max retries exceeded with url: /api/v1/stats (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7feae88ec970>: Failed to establish a new connection: [Errno 111] Connection refused'))
ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [181223]

2
logs/monitoring.pid
View File

@ -1 +1 @@
181223 192741

51
main.py
View File

@ -192,6 +192,39 @@ DEFAULT_ENDPOINTS = [
"POST /api/v1/run_plugin_gen_image", "POST /api/v1/run_plugin_gen_image",
] ]
API_ERROR_LOG_PATH = os.path.join(log_dir, "api_errors.jsonl")
API_ERROR_MAX_BYTES = 10 * 1024 * 1024 # 10MB
API_ERROR_BACKUP_COUNT = 5
def _rotate_api_error_log_if_needed():
try:
if os.path.exists(API_ERROR_LOG_PATH) and os.path.getsize(API_ERROR_LOG_PATH) >= API_ERROR_MAX_BYTES:
ts = time.strftime("%Y%m%d-%H%M%S")
rotated_path = os.path.join(log_dir, f"api_errors_{ts}.jsonl")
os.replace(API_ERROR_LOG_PATH, rotated_path)
# 오래된 로테이션 파일 정리 (최신 N개만 유지)
rotated = [
os.path.join(log_dir, f) for f in os.listdir(log_dir)
if f.startswith("api_errors_") and f.endswith(".jsonl")
]
rotated.sort(key=lambda p: os.path.getmtime(p), reverse=True)
for old in rotated[API_ERROR_BACKUP_COUNT:]:
try:
os.remove(old)
except Exception:
pass
except Exception as e: # pragma: no cover
logger.warning(f"API 에러 로그 로테이션 실패: {e}")
def _append_api_error_log(record: dict):
"""에러 전용 JSONL 로그에 한 줄 추가"""
try:
_rotate_api_error_log_if_needed()
with open(API_ERROR_LOG_PATH, "a", encoding="utf-8") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
except Exception as e: # pragma: no cover
logger.warning(f"API 에러 로그 기록 실패: {e}")
async def save_status_periodically(): async def save_status_periodically():
"""주기적으로 워커와 세션 상태를 파일에 저장합니다.""" """주기적으로 워커와 세션 상태를 파일에 저장합니다."""
logger.info("🔄 상태 저장 백그라운드 작업 시작됨") logger.info("🔄 상태 저장 백그라운드 작업 시작됨")
@ -341,6 +374,16 @@ async def collect_api_stats(request: Request, call_next):
# 통계 업데이트 # 통계 업데이트
api_stats.end_request(endpoint, success, response_time) api_stats.end_request(endpoint, success, response_time)
# 4xx/5xx는 에러 로그 파일에 기록
if not success:
_append_api_error_log({
"timestamp": time.time(),
"method": request.method,
"path": path,
"status": response.status_code,
"response_time_ms": int(response_time * 1000)
})
return response return response
@ -348,6 +391,14 @@ async def collect_api_stats(request: Request, call_next):
# 에러 발생 시 # 에러 발생 시
response_time = time.time() - start_time response_time = time.time() - start_time
api_stats.end_request(endpoint, False, response_time, str(e)) api_stats.end_request(endpoint, False, response_time, str(e))
_append_api_error_log({
"timestamp": time.time(),
"method": request.method,
"path": path,
"status": 500,
"error": str(e),
"response_time_ms": int(response_time * 1000)
})
raise raise
# CORS 미들웨어 추가 # CORS 미들웨어 추가

8
status.json
View File

@ -6,14 +6,14 @@
"workers_by_status": { "workers_by_status": {
"idle": [ "idle": [
{ {
"id": "worker_102e7692", "id": "worker_eac127f0",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
"last_task_at": null "last_task_at": null
}, },
{ {
"id": "worker_e36b0a34", "id": "worker_3fb51aaf",
"status": "idle", "status": "idle",
"task_count": 0, "task_count": 0,
"error_count": 0, "error_count": 0,
@ -84,8 +84,8 @@
"current_concurrent": 0, "current_concurrent": 0,
"max_concurrent": 0, "max_concurrent": 0,
"requests_per_second": 0.0, "requests_per_second": 0.0,
"uptime": 55.028260707855225, "uptime": 5.007766008377075,
"recent_errors": [] "recent_errors": []
}, },
"timestamp": 1759303574.1276805 "timestamp": 1759306699.0522153
} }