diff --git a/readme.md b/readme.md index 2586ad8..c34791e 100644 --- a/readme.md +++ b/readme.md @@ -244,6 +244,7 @@ for epoch in range(num_epochs + 1): if epoch % 10 == 0: # 모델 저장 torch.save(cnn_wdi.state_dict(), 'CNN_WDI_' + str(epoch) + 'epoch.pth') +``` ----- @@ -251,6 +252,219 @@ for epoch in range(num_epochs + 1): * 모델의 성능지표(Precision, Recall, Accuracy, F1-Score)를 혼동행렬(Confusion Metrix)로 구현한다. +# Confusion Metrix + + +```python +import numpy as np +import matplotlib.pyplot as plt +import seaborn as sns +from sklearn.metrics import classification_report, confusion_matrix +import pandas as pd + +def plot_metrics(title, class_names, precisions, recalls, f1_scores, acc): + num_classes = len(class_names) + index = np.arange(num_classes) + bar_width = 0.2 + + plt.figure(figsize=(15, 7)) + plt.bar(index, precisions, bar_width, label='Precision') + plt.bar(index + bar_width, recalls, bar_width, label='Recall') + plt.bar(index + 2 * bar_width, f1_scores, bar_width, label='F1-score') + plt.axhline(y=acc, color='r', linestyle='--', label='Accuracy') + + plt.xlabel('Class') + plt.ylabel('Scores') + plt.title(title + ': Precision, Recall, F1-score, and Accuracy per Class') + plt.xticks(index + bar_width, class_names) + plt.legend(loc='upper right') + plt.show() + +def predict_and_plot_metrics(title, model, dataloader, criterion, device): + model.eval() + running_loss = 0.0 + running_corrects = 0 + + all_preds = [] + all_labels = [] + class_names = ['Center', 'Donut', 'Edge-Loc', 'Edge-Ring', 'Loc', 'Near-full', 'none', 'Random', 'Scratch'] + + with torch.no_grad(): + for inputs, labels in dataloader: + inputs = inputs.to(device) + labels = labels.to(device) + + outputs = model(inputs) + _, preds = torch.max(outputs, 1) + loss = criterion(outputs, labels) + + running_loss += loss.item() * inputs.size(0) + running_corrects += torch.sum(preds == labels.data) + + all_preds.extend(preds.cpu().numpy()) + all_labels.extend(labels.cpu().numpy()) + + epoch_loss = running_loss / len(dataloader.dataset) + epoch_acc = running_corrects.double() / len(dataloader.dataset) + + + # Calculate classification report + report = classification_report(all_labels, all_preds, target_names=class_names, output_dict=True) + + # Calculate confusion matrix + cm = confusion_matrix(all_labels, all_preds) + + # Calculate precision, recall, and f1-score per class + precisions = [report[c]['precision'] for c in class_names] + recalls = [report[c]['recall'] for c in class_names] + f1_scores = [report[c]['f1-score'] for c in class_names] + print('p: ' + str(precisions)) + print('r: ' + str(recalls)) + print('f: ' + str(f1_scores)) + + # Plot confusion matrix with normalized values (percentage) + cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + plt.figure(figsize=(12, 12)) + sns.heatmap(cm_normalized, annot=True, fmt='.2%', cmap='Blues', xticklabels=class_names, yticklabels=class_names) + plt.xlabel('Predicted Label') + plt.ylabel('True Label') + plt.title('Normalized Confusion Matrix: ' + title) + plt.show() + + # Plot precision, recall, f1-score, and accuracy per class + plot_metrics(title, class_names, precisions, recalls, f1_scores, epoch_acc.item()) + + return epoch_loss, epoch_acc, report + +``` + +# Evaluate + + +```python +import os +import re + +test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=112, shuffle=False, num_workers=4) + +dir = '.' +models = [file for file in os.listdir(dir) if file.endswith(('.pth'))] + +def extract_number(filename): + return int(re.search(r'\d+', filename).group(0)) + +sorted_models = sorted(models, key=extract_number) + +for model in sorted_models: + model_path = os.path.join(dir, model) + + # Load the saved model weights + cnn_wdi.load_state_dict(torch.load(model_path)) + + # Call the predict_and_plot_metrics function with the appropriate arguments + epoch_loss, epoch_acc, report = predict_and_plot_metrics(model, cnn_wdi, test_loader, criterion, device) + # print(f'Model: {model} Test Loss: {test_loss:.4f} Acc: {test_acc:.4f}') +``` + +# Loss Graph + + +```python +import matplotlib.pyplot as plt + +# 파일에서 데이터를 읽어들입니다. +with open('output.txt', 'r') as file: + lines = file.readlines()[1:] # 첫 번째 줄은 헤더이므로 건너뜁니다. + +# 데이터를 분석하여 리스트에 저장합니다. +epochs = [] +train_losses = [] +train_accuracies = [] +val_losses = [] +val_accuracies = [] + +for line in lines: + if line == '\n': + continue + # epoch, train_loss, train_acc, val_loss, val_acc = line.strip().split(', \t') + epoch, train_loss, train_acc, val_loss, val_acc = re.split(r'[,\s\t]+', line.strip()) + epochs.append(int(epoch.split('/')[0])) + train_losses.append(float(train_loss)) + train_accuracies.append(float(train_acc)) + val_losses.append(float(val_loss)) + val_accuracies.append(float(val_acc)) + +# 선 그래프를 그립니다. +plt.figure(figsize=(10, 5)) + +plt.plot(epochs, train_losses, label='Train Loss') +plt.plot(epochs, train_accuracies, label='Train Acc') +plt.plot(epochs, val_losses, label='Val Loss') +plt.plot(epochs, val_accuracies, label='Val Acc') + +plt.xlabel('Epochs') +plt.ylabel('Values') +plt.title('Training and Validation Loss and Accuracy') +plt.legend() +plt.show() + +``` + +# Print Selecting Test Model Result + + +```python +def output(model, dataloader, criterion, device): + model.eval() + running_loss = 0.0 + running_corrects = 0 + + all_preds = [] + all_labels = [] + class_names = ['Center', 'Donut', 'Edge-Loc', 'Edge-Ring', 'Loc', 'Near-full', 'none', 'Random', 'Scratch'] + + with torch.no_grad(): + for inputs, labels in dataloader: + inputs = inputs.to(device) + labels = labels.to(device) + + outputs = model(inputs) + _, preds = torch.max(outputs, 1) + loss = criterion(outputs, labels) + + running_loss += loss.item() * inputs.size(0) + running_corrects += torch.sum(preds == labels.data) + + all_preds.extend(preds.cpu().numpy()) + all_labels.extend(labels.cpu().numpy()) + + epoch_loss = running_loss / len(dataloader.dataset) + epoch_acc = running_corrects.double() / len(dataloader.dataset) + + + # Calculate classification report + report = classification_report(all_labels, all_preds, target_names=class_names, output_dict=True) + + # Calculate precision, recall, and f1-score per class + precisions = [report[c]['precision'] for c in class_names] + recalls = [report[c]['recall'] for c in class_names] + f1_scores = [report[c]['f1-score'] for c in class_names] + accuracy = report['accuracy'] + + precs = sum(precisions) / len(precisions) + recs = sum(recalls) / len(recalls) + f1s = sum(f1_scores) / len(f1_scores) + print('precisions: ' + str(precs)) + print('recalls: ' + str(recs)) + print('f1_scores: ' + str(f1s)) + print('accuracy ' + str(accuracy)) + + +selected_model = 'CNN_WDI_20epoch.pth' +cnn_wdi.load_state_dict(torch.load(selected_model)) +output(cnn_wdi, test_loader, criterion, device) +``` + ----- ### 테스트 결과