Spaces:
Sleeping
Sleeping
| import os | |
| import pandas as pd | |
| import numpy as np | |
| compare_to_gt = True | |
| ground_truth = pd.read_csv("data/zipfiles.csv", sep="\t") | |
| results = pd.read_csv("data/results.csv", sep="\t") | |
| verbose = 0 | |
| eval_readme = [] | |
| eval_training = [] | |
| eval_evaluating = [] | |
| eval_licensing = [] | |
| eval_weights = [] | |
| eval_dependencies = [] | |
| full_results = [] | |
| for (index1, row1), (index2, row2) in zip(ground_truth.iterrows(), results.iterrows()): | |
| if (pd.isna(row1["training"])): | |
| continue | |
| print(f"\nEvaluating {index1+1} out of {len(ground_truth.index)} papers...") | |
| print(f'Paper title - "{row1["title"]}" ({row1["year"]})') | |
| print(f'Repository link - {row1["url"]}') | |
| if ((not(pd.isna(row1["dependencies"]))) & (row2["pred_dependencies"] is not None)): | |
| eval_dependencies.append(row2["pred_dependencies"] == row1["dependencies"]) | |
| if (row2["pred_dependencies"] != row1["dependencies"]): | |
| print(f"Dependencies acc. - {row2['pred_dependencies']} (GT:{row1['dependencies']})") | |
| if ((not(pd.isna(row1["training"]))) & (row2["pred_dependencies"] is not None)): | |
| eval_training.append(row1["training"] == row2["pred_training"]) | |
| if (row1["training"] != row2["pred_training"]): | |
| print(f"Training acc. -{row2['pred_training']} (GT:{row1['training']})") | |
| if ((not(pd.isna(row1["evaluation"]))) & (row2["pred_dependencies"] is not None)): | |
| eval_evaluating.append(row1["evaluation"] == row2["pred_evaluation"]) | |
| if (row1["evaluation"] != row2["pred_evaluation"]): | |
| print(f"Evaluating acc. - {row2['pred_evaluation']} (GT:{row1['evaluation']})") | |
| if ((not(pd.isna(row1["weights"]))) & (row2["pred_dependencies"] is not None)): | |
| eval_weights.append(row1["weights"] == row2["pred_weights"]) | |
| if (row1["weights"] != row2["pred_weights"]): | |
| print(f"Weights acc. - {row2['pred_weights']} (GT:{row1['weights']})") | |
| if ((not(pd.isna(row1["readme"]))) & (row2["pred_dependencies"] is not None)): | |
| eval_readme.append(row1["readme"] == row2["pred_readme"]) | |
| if (row1["readme"] != row2["pred_readme"]): | |
| print(f"README acc. - {row2['pred_readme']} (GT:{row1['readme']})") | |
| if ((not(pd.isna(row1["license"]))) & (row2["pred_dependencies"] is not None)): | |
| eval_licensing.append(("No" if row1["license"] == "No" else "Yes") == row2["pred_license"]) | |
| if (("No" if row1["license"] == "No" else "Yes") != row2["pred_license"]): | |
| print(f"LICENSE acc. - {row2['pred_license']} (GT:{row1['license']})") | |
| print("\nSummary:") | |
| print(f"Dependencies acc. - {int(100 * np.mean(eval_dependencies))}%") | |
| print(f"Training acc. - {int(100 * np.mean(eval_training))}%") | |
| print(f"Evaluating acc. - {int(100 * np.mean(eval_evaluating))}%") | |
| print(f"Weights acc. - {int(100 * np.mean(eval_weights))}%") | |
| print(f"README acc. - {int(100 * np.mean(eval_readme))}%") | |
| print(f"LICENSE acc. - {int(100 * np.mean(eval_licensing))}%") |