new evaluations

.gitignore

@@ -1,5 +1,10 @@
-data/*.csv
+data/MIDL.csv
+data/MICCAI.csv
+data/arXiv.csv
+data/Nature.csv
+data/results.csv
 data/*.zip
+data/test/*
 *.env
 .env
 evaluations/__pycache__/*

data/fetch_processed.py

@@ -9,6 +9,7 @@ custom_order = ["MICCAI", "MIDL", "Nature", "arXiv"]
 
 for venue in custom_order:
     df = pd.read_excel("https://docs.google.com/spreadsheets/d/e/2PACX-1vQjpsSYcEcYUVB-88bCQ01UfQf0z9m16ax7p1ft03G68Nr-DdXHpPt-xOFSrXFj1N49AjK5nYhmKBfo/pub?output=xlsx", sheet_name=venue)
+    df = df.replace('\t', ' ', regex=True)
     df.to_csv(f'data/{venue}.csv', sep="\t")
 
 # Store all evaluations here

data/fetch_zips.py

@@ -0,0 +1,43 @@
+import csv
+import numpy as np
+import sys
+import pandas as pd
+import re
+sys.path.append("./")
+from evaluations.utils import *
+
+token = os.getenv("githubToken")
+custom_order = ["MICCAI", "MIDL", "Nature", "arXiv"]
+
+for venue in custom_order:
+    df = pd.read_excel("https://docs.google.com/spreadsheets/d/e/2PACX-1vQjpsSYcEcYUVB-88bCQ01UfQf0z9m16ax7p1ft03G68Nr-DdXHpPt-xOFSrXFj1N49AjK5nYhmKBfo/pub?output=xlsx", sheet_name=venue)
+    df = df.replace('\t', ' ', regex=True)
+    df.to_csv(f'data/{venue}.csv', sep="\t")
+
+# Store all evaluations here
+paper_dump = pd.DataFrame()
+# Official color codes for conferences
+zip_idx = 0
+
+for venue in custom_order:
+    with open(f'data/{venue}.csv') as file:
+        tsv_file = csv.reader(file, delimiter="\t")
+        for row in tsv_file:
+            if (row[0] == ""):
+                continue
+
+            if (row[1] == ""):
+                continue
+
+            repo_url = row[4]
+            username, repo_name = decompose_url(repo_url)
+            repo_save_name = f"repo_{zip_idx}.zip"
+            repository_zip_name = f"data/test/{repo_save_name}"
+            log(0, "LOG", f"Fetching github repository: https://github.com/{username}/{repo_name}")
+            fetch_repo(0, repo_url, repository_zip_name, token)
+
+            if (os.path.exists(repository_zip_name)):
+                paper_dump = pd.concat([paper_dump, pd.DataFrame({"venue": venue, "title": [row[1]], "year": [row[2]], "pdf": [row[3]], "url": [row[4]], "public": [row[5]], "dependencies": [row[6]], "training": [row[7]], "evaluation": [row[8]], "weights": [row[9]], "readme": [row[10]], "license": [row[11]], "zip_idx": [ repository_zip_name ]})], ignore_index=True)
+                zip_idx += 1
+            
+paper_dump.to_csv(f'data/zipfiles.csv', sep="\t")

evaluations/repo_evaluations.py

@@ -7,10 +7,7 @@ import os
 import numpy as np
 from huggingface_hub import InferenceClient
 
-def evaluate(llm, verbose, repo_url, title=None, year=None):
-    repository_zip_name = "data/repo.zip"
-    token = os.getenv("githubToken")
-
+def evaluate(llm, verbose, repo_url, title=None, year=None, zip=None):
     try:
         if (not(llm)):
             log(verbose, "LOG", "No LLM will be used for the evaluation.")
@@ -31,20 +28,26 @@ def evaluate(llm, verbose, repo_url, title=None, year=None):
             return results
 
         username, repo_name = decompose_url(repo_url)
-        log(verbose, "LOG", f"Fetching github repository: https://github.com/{username}/{repo_name}")
 
-        fetch_repo(verbose, repo_url, repository_zip_name, token)
+        # If you don't provide a zip file, it will be fetched from github. For this, you need to provide a github token.
+        if (zip is None):
+            token = os.getenv("githubToken")
+            repository_zip_name = "data/repo.zip"
+            log(verbose, "LOG", f"Fetching github repository: https://github.com/{username}/{repo_name}")
 
+            fetch_repo(verbose, repo_url, repository_zip_name, token)
 
-        if (not(os.path.exists(repository_zip_name))):
-            results["pred_live"] = "No"
-            return results
+            if (not(os.path.exists(repository_zip_name))):
+                results["pred_live"] = "No"
+                return results
+            
+            results["pred_stars"] = fetch_repo_stars(verbose, repo_url, token)
+
+            zip = zipfile.ZipFile(repository_zip_name)
 
-        zip = zipfile.ZipFile(repository_zip_name)
         readme = fetch_readme(zip)
         results["NA"] = documentation.is_applicable(verbose, llm, readme)
 
-        results["pred_stars"] = fetch_repo_stars(verbose, repo_url, token)
         results["pred_license"] = license.evaluate(verbose, llm, zip, readme)
 
         if (len(zip.namelist()) <= 2):
@@ -65,7 +68,7 @@ def evaluate(llm, verbose, repo_url, title=None, year=None):
         return results
 
 def full_evaluation():
-  paper_dump = pd.read_csv("data/dump.csv", sep="\t")
+  paper_dump = pd.read_csv("data/zipfiles.csv", sep="\t")
   full_results = []
 
   for idx, row in paper_dump.iterrows():
@@ -74,62 +77,9 @@ def full_evaluation():
           continue
 
       print(str(int(100 * idx / paper_dump["title"].count())) + "% done")
-      result = evaluate(None, False, row["url"], row["title"], row["year"])
+      result = evaluate(None, False, row["url"], row["title"], row["year"], zip=zipfile.ZipFile(row["zip_idx"]))
       for column in result.keys():
           row[column] = result[column]
 
       full_results.append(row)
   return pd.DataFrame(full_results)
-
-def midl_evaluations(model):
-  compare_to_gt = True
-  paper_dump = pd.read_csv("data/dump.csv", sep="\t")
-  verbose = 0
-
-  eval_readme = []
-  eval_training = []
-  eval_evaluating = []
-  eval_licensing = []
-  eval_weights = []
-  eval_dependencies = []
-  full_results = []
-  for idx, row in paper_dump.iterrows():
-      if (row["venue"] != "MIDL"):
-          continue
-
-      if (row["venue"] == 2024):
-          continue
-
-      if (pd.isna(row["url"]) | (row["url"] == "")):
-          continue
-
-      print(f"\nEvaluating {idx+1} out of {len(paper_dump.index)} papers...")
-      print(f'Paper title - "{row["title"]}" ({row["year"]})')
-      print(f'Repository link - {row["url"]}')
-      result = evaluate(model, verbose, row["url"])
-      for column in result.keys():
-          row[column] = result[column]
-      full_results.append(row)
-      if (compare_to_gt):
-          print("\nSummary:")
-          print(row["NA"])
-          if ((~pd.isna(row["dependencies"])) & (row["pred_dependencies"] is not None)):
-              eval_dependencies.append(row["pred_dependencies"] == row["dependencies"])
-              print(f"Dependencies acc. - {row['pred_dependencies']} (GT:{row['dependencies']}) / {int(100 * np.mean(eval_dependencies))}%")
-          if ((~pd.isna(row["training"])) & (row["pred_dependencies"] is not None)):
-              eval_training.append(row["training"] == row["pred_training"])
-              print(f"Training acc. -{row['pred_training']} (GT:{row['training']}) / {int(100 * np.mean(eval_training))}%")
-          if ((~pd.isna(row["evaluation"])) & (row["pred_dependencies"] is not None)):
-              eval_evaluating.append(row["evaluation"] == row["pred_evaluation"])
-              print(f"Evaluating acc. - {row['pred_evaluation']} (GT:{row['evaluation']}) / {int(100 * np.mean(eval_evaluating))}%")
-          if ((~pd.isna(row["weights"])) & (row["pred_dependencies"] is not None)):
-              eval_weights.append(row["weights"] == row["pred_weights"])
-              print(f"Weights acc. - {row['pred_weights']} (GT:{row['weights']}) / {int(100 * np.mean(eval_weights))}%")
-          if ((~pd.isna(row["readme"])) & (row["pred_dependencies"] is not None)):
-              eval_readme.append(row["readme"] == row["pred_readme"])
-              print(f"README acc. - {row['pred_readme']} (GT:{row['readme']}) / {int(100 * np.mean(eval_readme))}%")
-          if ((~pd.isna(row["license"])) & (row["pred_dependencies"] is not None)):
-              eval_licensing.append(("No" if row["license"] == "No" else "Yes") == row["pred_license"])
-              print(f"LICENSE acc. - {row['pred_license']} (GT:{row['license']}) / {int(100 * np.mean(eval_licensing))}%")
-
-  return pd.DataFrame(full_results)

evaluations/utils.py

@@ -51,7 +51,6 @@ def fetch_repo(verbose, repo_url, repo_name, token):
     if (os.path.exists(repo_name)):
         os.remove(repo_name)
 
-
     if ("github.com" not in repo_url):
         log(verbose, "ERROR", f"URL not for github repo, please evaluate manually ({repo_url}).")
         return

full_eval.py

@@ -1,11 +1,4 @@
 from evaluations.repo_evaluations import full_evaluation
-# importing os module for environment variables
-import os
-# importing necessary functions from dotenv library
-from dotenv import load_dotenv 
-# loading variables from .env file
-load_dotenv() 
-token = os.getenv("githubToken")
 
 res = full_evaluation()
 res.to_csv("data/results.csv", sep="\t", index=False)

midl.py

@@ -1,12 +0,0 @@
-from evaluations.repo_evaluations import midl_evaluations
-from evaluations.models import LocalLLM
-import os
-from dotenv import load_dotenv
-load_dotenv() 
-token = os.getenv("githubToken")
-
-# Load model directly
-model = LocalLLM("meta-llama/Llama-3.2-3B-Instruct")
-
-res = midl_evaluations(model)
-res.to_csv("results_midl.csv", sep="\t", index=False)

midl_summary.py

@@ -0,0 +1,57 @@
+import os
+import pandas as pd
+import numpy as np
+
+
+compare_to_gt = True
+ground_truth = pd.read_csv("data/zipfiles.csv", sep="\t")
+results = pd.read_csv("data/results.csv", sep="\t")
+verbose = 0
+
+eval_readme = []
+eval_training = []
+eval_evaluating = []
+eval_licensing = []
+eval_weights = []
+eval_dependencies = []
+full_results = []
+for (index1, row1), (index2, row2) in zip(ground_truth.iterrows(), results.iterrows()):
+    if (pd.isna(row1["training"])):
+        continue
+
+    print(f"\nEvaluating {index1+1} out of {len(ground_truth.index)} papers...")
+    print(f'Paper title - "{row1["title"]}" ({row1["year"]})')
+    print(f'Repository link - {row1["url"]}')
+    if ((not(pd.isna(row1["dependencies"]))) & (row2["pred_dependencies"] is not None)):
+        eval_dependencies.append(row2["pred_dependencies"] == row1["dependencies"])
+        if (row2["pred_dependencies"] != row1["dependencies"]):
+            print(f"Dependencies acc. - {row2['pred_dependencies']} (GT:{row1['dependencies']})")
+    if ((not(pd.isna(row1["training"]))) & (row2["pred_dependencies"] is not None)):
+        eval_training.append(row1["training"] == row2["pred_training"])
+        if (row1["training"] != row2["pred_training"]):
+            print(f"Training acc. -{row2['pred_training']} (GT:{row1['training']})")
+    if ((not(pd.isna(row1["evaluation"]))) & (row2["pred_dependencies"] is not None)):
+        eval_evaluating.append(row1["evaluation"] == row2["pred_evaluation"])
+        if (row1["evaluation"] != row2["pred_evaluation"]):
+            print(f"Evaluating acc. - {row2['pred_evaluation']} (GT:{row1['evaluation']})")
+    if ((not(pd.isna(row1["weights"]))) & (row2["pred_dependencies"] is not None)):
+        eval_weights.append(row1["weights"] == row2["pred_weights"])
+        if (row1["weights"] != row2["pred_weights"]):
+            print(f"Weights acc. - {row2['pred_weights']} (GT:{row1['weights']})")
+    if ((not(pd.isna(row1["readme"]))) & (row2["pred_dependencies"] is not None)):
+        eval_readme.append(row1["readme"] == row2["pred_readme"])
+        if (row1["readme"] != row2["pred_readme"]):
+            print(f"README acc. - {row2['pred_readme']} (GT:{row1['readme']})")
+    if ((not(pd.isna(row1["license"]))) & (row2["pred_dependencies"] is not None)):
+        eval_licensing.append(("No" if row1["license"] == "No" else "Yes") == row2["pred_license"])
+        if (("No" if row1["license"] == "No" else "Yes") != row2["pred_license"]):
+            print(f"LICENSE acc. - {row2['pred_license']} (GT:{row1['license']})")
+
+
+print("\nSummary:")
+print(f"Dependencies acc. - {int(100 * np.mean(eval_dependencies))}%")
+print(f"Training acc. - {int(100 * np.mean(eval_training))}%")
+print(f"Evaluating acc. - {int(100 * np.mean(eval_evaluating))}%")
+print(f"Weights acc. - {int(100 * np.mean(eval_weights))}%")
+print(f"README acc. - {int(100 * np.mean(eval_readme))}%")
+print(f"LICENSE acc. - {int(100 * np.mean(eval_licensing))}%")