ckh08045
/
inpaintServer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

상태 JSON 파일의 API 요청 통계를 대폭 업데이트하고, 대시보드에 세션 풀 사용량 및 워커 활성도 차트를 추가하였습니다. 실시간 상태 업데이트 기능을 개선하고, 서버 로그에서 프로세스 ID 변경 사항을 반영하였습니다.

This commit is contained in:
vast 2025-10-02 07:06:00 +00:00
parent bd7e4b4b33
commit f9a3f2de61
6 changed files with 66954 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
app/monitoring/dashboard.py
View File

@ -1198,6 +1198,78 @@ HTML_TEMPLATE = """
}
});
// 세션 풀 사용량 차트
const sessionCtx = document.getElementById('sessionChart').getContext('2d');
const sessionChart = new Chart(sessionCtx, {
type: 'line',
data: {
labels: [],
datasets: [{
label: 'Simple LAMA 사용중',
data: [],
borderColor: 'rgb(255, 99, 132)',
backgroundColor: 'rgba(255, 99, 132, 0.1)',
tension: 0.1
}, {
label: 'MIGAN 사용중',
data: [],
borderColor: 'rgb(54, 162, 235)',
backgroundColor: 'rgba(54, 162, 235, 0.1)',
tension: 0.1
}, {
label: 'RemBG 사용중',
data: [],
borderColor: 'rgb(75, 192, 192)',
backgroundColor: 'rgba(75, 192, 192, 0.1)',
tension: 0.1
}]
},
options: {
responsive: true,
scales: {
y: {
beginAtZero: true,
ticks: {
stepSize: 1
}
}
}
}
});
// 워커 활성도 차트
const workerCtx = document.getElementById('workerChart').getContext('2d');
const workerChart = new Chart(workerCtx, {
type: 'line',
data: {
labels: [],
datasets: [{
label: 'Idle 워커',
data: [],
borderColor: 'rgb(75, 192, 192)',
backgroundColor: 'rgba(75, 192, 192, 0.1)',
tension: 0.1
}, {
label: 'Busy 워커',
data: [],
borderColor: 'rgb(255, 159, 64)',
backgroundColor: 'rgba(255, 159, 64, 0.1)',
tension: 0.1
}]
},
options: {
responsive: true,
scales: {
y: {
beginAtZero: true,
ticks: {
stepSize: 1
}
}
}
}
});
// WebSocket 연결 관리
let ws;
let reconnectAttempts = 0;
@ -1398,6 +1470,27 @@ HTML_TEMPLATE = """
document.getElementById('server-uptime').textContent = `${hours}h ${minutes}m ${seconds}s`;
}
// 일일 통계 업데이트
if (data.daily_stats) {
const totalImages = data.daily_stats.images_processed?.total || 0;
document.getElementById('daily-images-total').textContent = totalImages.toLocaleString();
const breakdown = [
`inpaint: ${data.daily_stats.images_processed?.inpaint || 0}`,
`remove_bg: ${data.daily_stats.images_processed?.remove_bg || 0}`,
`gen_image: ${data.daily_stats.images_processed?.gen_image || 0}`
].join(', ');
document.getElementById('daily-images-breakdown').textContent = breakdown;
const uploadGB = (data.daily_stats.network?.gb_uploaded || 0).toFixed(2);
document.getElementById('daily-upload-size').textContent = `${uploadGB} GB`;
const downloadGB = (data.daily_stats.network?.gb_downloaded || 0).toFixed(2);
document.getElementById('daily-download-size').textContent = `${downloadGB} GB`;
document.getElementById('daily-peak-concurrent').textContent = data.daily_stats.peak_concurrent || 0;
}
// 시스템 성능 상세 업데이트
if (data.system_performance?.cpu) {
document.getElementById('cpu-count').textContent = data.system_performance.cpu.count || '-';
@ -1637,8 +1730,43 @@ HTML_TEMPLATE = """
gpuChart.data.datasets[0].data.shift();
}
// 세션 풀 사용량 차트 업데이트
console.log('Sessions data:', data.sessions); // 디버그
sessionChart.data.labels.push(timestamp);
const lamaInUse = data.sessions?.simple_lama?.in_use || 0;
const miganInUse = data.sessions?.migan?.in_use || 0;
const rembgInUse = data.sessions?.rembg?.in_use || 0;
console.log('Session in_use:', {lama: lamaInUse, migan: miganInUse, rembg: rembgInUse}); // 디버그
sessionChart.data.datasets[0].data.push(lamaInUse);
sessionChart.data.datasets[1].data.push(miganInUse);
sessionChart.data.datasets[2].data.push(rembgInUse);
if (sessionChart.data.labels.length > 20) {
sessionChart.data.labels.shift();
sessionChart.data.datasets[0].data.shift();
sessionChart.data.datasets[1].data.shift();
sessionChart.data.datasets[2].data.shift();
}
// 워커 활성도 차트 업데이트
console.log('Workers data:', data.workers); // 디버그
workerChart.data.labels.push(timestamp);
const idleWorkers = data.workers?.workers_by_status?.idle?.length || 0;
const busyWorkers = data.workers?.workers_by_status?.busy?.length || 0;
console.log('Worker counts:', {idle: idleWorkers, busy: busyWorkers}); // 디버그
workerChart.data.datasets[0].data.push(idleWorkers);
workerChart.data.datasets[1].data.push(busyWorkers);
if (workerChart.data.labels.length > 20) {
workerChart.data.labels.shift();
workerChart.data.datasets[0].data.shift();
workerChart.data.datasets[1].data.shift();
}
performanceChart.update();
gpuChart.update();
sessionChart.update();
workerChart.update();
}
function updateModelPerformance(stats) {

18157
logs/main.log
View File

File diff suppressed because it is too large Load Diff

48590
logs/main_server.log
View File

File diff suppressed because it is too large Load Diff

77
logs/monitoring.log
View File

@ -1,49 +1,38 @@
WARNING:root:jtop library not found. Jetson monitoring will be limited. Please run 'sudo pip install jetson-stats'
INFO: Started server process [326863]
INFO: Started server process [342486]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:8888 (Press CTRL+C to quit)
INFO: 122.35.47.45:59571 - "WebSocket /ws" [accepted]
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 127.0.0.1:47424 - "GET /api/simple HTTP/1.1" 200 OK
INFO: 122.35.47.45:64056 - "WebSocket /ws" [accepted]
INFO: connection open
WARNING:app.monitoring.dashboard:실시간 상태 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 122.35.47.45:59569 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59567 - "GET / HTTP/1.1" 200 OK
INFO: 122.35.47.45:59574 - "GET / HTTP/1.1" 200 OK
INFO: 122.35.47.45:59575 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 127.0.0.1:56324 - "GET /api/simple HTTP/1.1" 200 OK
INFO: 122.35.47.45:59578 - "GET / HTTP/1.1" 200 OK
INFO: 122.35.47.45:59569 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59577 - "WebSocket /ws" [accepted]
INFO: connection open
INFO: 122.35.47.45:59576 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:59568 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:59578 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:59575 - "GET /api/performance-stats HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: 122.35.47.45:59584 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: connection closed
INFO: connection closed
INFO: 122.35.47.45:59592 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59575 - "GET / HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: 122.35.47.45:59588 - "WebSocket /ws" [accepted]
INFO: connection open
INFO: 122.35.47.45:59592 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:59575 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:59604 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:59603 - "GET /api/system-alerts HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:데이터 전송 오류:
INFO: connection closed
INFO: 122.35.47.45:59578 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:59602 - "WebSocket /ws" [accepted]
INFO: connection open
INFO: 122.35.47.45:59584 - "GET /api/errors HTTP/1.1" 200 OK
ERROR:app.monitoring.dashboard:모델 성능 통계 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
WARNING:app.monitoring.dashboard:실시간 상태 조회 중 예외 발생: HTTPConnectionPool(host='0.0.0.0', port=8008): Read timed out. (read timeout=2)
INFO: 122.35.47.45:59624 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59625 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:59630 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:59629 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:59629 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:59630 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:64067 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64068 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64103 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64104 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:64105 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64102 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:64109 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64168 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64167 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64169 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:64166 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64170 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:64241 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64244 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:64246 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64242 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64243 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64245 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:64297 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64296 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:64299 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64294 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64295 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64298 - "GET /api/errors HTTP/1.1" 200 OK
INFO: 122.35.47.45:64346 - "GET /api/model-usage-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64347 - "GET /api/system-alerts HTTP/1.1" 200 OK
INFO: 122.35.47.45:64349 - "GET /api/logs?lines=50 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64343 - "GET /api/performance-stats HTTP/1.1" 200 OK
INFO: 122.35.47.45:64345 - "GET /api/session_events?limit=100 HTTP/1.1" 200 OK
INFO: 122.35.47.45:64344 - "GET /api/errors HTTP/1.1" 200 OK

2
logs/monitoring.pid
View File

@ -1 +1 @@
326863
342486

90
status.json
View File

@ -78,43 +78,43 @@
}
},
"api_stats": {
"total_requests": 95,
"successful_requests": 95,
"total_requests": 9479,
"successful_requests": 9479,
"failed_requests": 0,
"success_rate": 100.0,
"endpoint_usage": {
"GET /api/v1/model": 44,
"POST /api/v1/run_plugin_gen_image": 5,
"POST /api/v1/inpaint": 26,
"GET /api/v1/realtime_status": 20
"GET /api/v1/model": 3354,
"POST /api/v1/run_plugin_gen_image": 298,
"POST /api/v1/inpaint": 3040,
"GET /api/v1/realtime_status": 2787
},
"endpoint_stats": {
"GET /api/v1/model": {
"count": 44,
"avg_time": 0.00349293513731523,
"min_time": 0.0007824897766113281,
"max_time": 0.008572578430175781,
"count": 3354,
"avg_time": 0.001591782569885254,
"min_time": 0.0004963874816894531,
"max_time": 0.003871917724609375,
"current_concurrent": 0
},
"POST /api/v1/run_plugin_gen_image": {
"count": 5,
"avg_time": 1.928318691253662,
"min_time": 0.2756519317626953,
"max_time": 8.236981391906738,
"current_concurrent": 0
"count": 298,
"avg_time": 0.6214150261878967,
"min_time": 0.1577005386352539,
"max_time": 9.059064388275146,
"current_concurrent": 1
},
"POST /api/v1/inpaint": {
"count": 26,
"avg_time": 1.5387938297711885,
"min_time": 0.4340169429779053,
"max_time": 7.625817060470581,
"count": 3040,
"avg_time": 0.6592595100402832,
"min_time": 0.3056967258453369,
"max_time": 2.1711950302124023,
"current_concurrent": 2
},
"GET /api/v1/realtime_status": {
"count": 20,
"avg_time": 0.002240335941314697,
"min_time": 0.0006320476531982422,
"max_time": 0.006146907806396484,
"count": 2787,
"avg_time": 0.0018257951736450194,
"min_time": 0.0007917881011962891,
"max_time": 0.004314422607421875,
"current_concurrent": 0
},
"POST /api/v1/remove_bg": {
@ -125,41 +125,41 @@
"current_concurrent": 0
}
},
"average_response_time": 0.524723462054604,
"min_response_time": 0.0006320476531982422,
"max_response_time": 8.236981391906738,
"current_concurrent": 2,
"max_concurrent": 7,
"requests_per_second": 1.6502782030794676,
"uptime": 57.5660514831543,
"average_response_time": 0.26329850912094116,
"min_response_time": 0.0004963874816894531,
"max_response_time": 9.481664896011353,
"current_concurrent": 3,
"max_concurrent": 10,
"requests_per_second": 2.134372224479969,
"uptime": 4441.1185131073,
"recent_errors": []
},
"daily_stats": {
"date": "2025-10-02",
"images_processed": {
"inpaint": 26,
"inpaint": 3040,
"remove_bg": 0,
"gen_image": 5,
"total": 31
"gen_image": 298,
"total": 3338
},
"network": {
"bytes_uploaded": 39333149,
"bytes_downloaded": 25089494,
"requests_count": 95,
"mb_uploaded": 37.511013984680176,
"mb_downloaded": 23.92720603942871,
"gb_uploaded": 0.036631849594414234,
"gb_downloaded": 0.0233664121478796
"bytes_uploaded": 3228232462,
"bytes_downloaded": 1961127959,
"requests_count": 9479,
"mb_uploaded": 3078.6823863983154,
"mb_downloaded": 1870.2773656845093,
"gb_uploaded": 3.006525767967105,
"gb_downloaded": 1.8264427399262786
},
"api_calls": {
"total": 95,
"success": 95,
"total": 9479,
"success": 9479,
"failed": 0
},
"models_used": {},
"peak_concurrent": 6,
"peak_concurrent": 9,
"start_time": 1759384315.930352,
"last_update": 1759384373.4347339
"last_update": 1759388756.0812397
},
"timestamp": 1759384373.5121431
"timestamp": 1759388757.0647824
}