new evaluations

evaluations/documentation.py

@@ -2,6 +2,17 @@ from .utils import log,fetch_code
 import re
 import numpy as np
 
+def is_applicable(verbose, llm, readme):
+    applicable = "NA/NA/NA"
+    if (llm):
+        log(verbose, "TITLE", "\nChecking what parts of the evaluations are applicable...")
+        res_training = llm.predict("STRICT", f"{readme}\nBased on the readme above, should the repository contain code for training a model?")
+        res_evaluation = llm.predict("STRICT", f"{readme}\nBased on the readme above, should the repository contain code for evaluating a model?")
+        res_weights = llm.predict("STRICT", f"{readme}\nBased on the readme above, should the repository contain code for loading pre-trained weights?")
+    
+    applicable = f"{res_training}/{res_evaluation}/{res_weights}"
+    return applicable
+
 def evaluate(verbose, llm, zip, readme):
   log(verbose, "TITLE", "\nEvaluating code documentation...")
   overall = "No"

evaluations/models.py

@@ -33,7 +33,7 @@ system_messages = { "STRICT": """You are a chatbot evaluating github repositorie
 
 class LocalLLM():
   def __init__(self, model_name):
-    self.pipe = pipeline("text-generation", model=model_name, max_new_tokens=1000, device_map={0: 0})
+    self.pipe = pipeline("text-generation", model=model_name, max_new_tokens=1000, device=0, pad_token_id=128001)
 
   def predict(self, response_type, prompt):
     messages = [

evaluations/repo_evaluations.py

@@ -8,59 +8,61 @@ import numpy as np
 from huggingface_hub import InferenceClient
 
 def evaluate(llm, verbose, repo_url, title=None, year=None):
-  repository_zip_name = "data/repo.zip"
-  token = os.getenv("githubToken")
+    repository_zip_name = "data/repo.zip"
+    token = os.getenv("githubToken")
 
-  if (not(llm)):
-      log(verbose, "LOG", "No LLM will be used for the evaluation.")
+    try:
+        if (not(llm)):
+            log(verbose, "LOG", "No LLM will be used for the evaluation.")
 
-  results = { "pred_live": "Yes", "pred_dependencies": None, "pred_training": None, "pred_evaluation": None, "pred_weights": None, "pred_readme": None, "pred_license": None, "pred_stars": None, "pred_citations": None, "pred_valid": False}
+        results = { "pred_live": "Yes", "pred_dependencies": None, "pred_training": None, "pred_evaluation": None, "pred_weights": None, "pred_readme": None, "pred_license": None, "pred_stars": None, "pred_citations": None, "pred_valid": False}
+        
+        if ((title != None) & (year != None) & (title != "") & (year != "")):
+            res = fetch_openalex(verbose, title, year)
+            if ((res != None)):
+                res = res["results"]
+                if (len(res) > 0):
+                    res = res[0]
+                    results["pred_citations"] = res["cited_by_count"]
 
-  try:
-      if (get_api_link(repo_url) != ""):
-          results["pred_valid"] = True
-      else:
-          return results
+        if (get_api_link(repo_url) != ""):
+            results["pred_valid"] = True
+        else:
+            return results
 
-      username, repo_name = decompose_url(repo_url)
-      log(verbose, "LOG", f"Fetching github repository: https://github.com/{username}/{repo_name}")
+        username, repo_name = decompose_url(repo_url)
+        log(verbose, "LOG", f"Fetching github repository: https://github.com/{username}/{repo_name}")
 
-      fetch_repo(verbose, repo_url, repository_zip_name, token)
+        fetch_repo(verbose, repo_url, repository_zip_name, token)
 
-      if ((title != None) & (year != None) & (title != "") & (year != "")):
-          res = fetch_openalex(verbose, title, year)
-          if (res != None):
-              res = res["results"]
-              if (len(res) > 0):
-                  res = res[0]
-                  results["pred_citations"] = res["cited_by_count"]
 
-      if (not(os.path.exists(repository_zip_name))):
-          results["pred_live"] = "No"
-          return results
+        if (not(os.path.exists(repository_zip_name))):
+            results["pred_live"] = "No"
+            return results
 
-      zip = zipfile.ZipFile(repository_zip_name)
-      readme = fetch_readme(zip)
+        zip = zipfile.ZipFile(repository_zip_name)
+        readme = fetch_readme(zip)
+        results["NA"] = documentation.is_applicable(verbose, llm, readme)
 
-      results["pred_stars"] = fetch_repo_stars(verbose, repo_url, token)
+        results["pred_stars"] = fetch_repo_stars(verbose, repo_url, token)
+        results["pred_license"] = license.evaluate(verbose, llm, zip, readme)
 
-      results["pred_license"] = license.evaluate(verbose, llm, zip, readme)
-      if (len(zip.namelist()) <= 2):
-          log(verbose, "LOG", "The repository is empty.")
+        if (len(zip.namelist()) <= 2):
+            log(verbose, "LOG", "The repository is empty.")
           
-      results["pred_dependencies"] = requirements.evaluate(verbose, llm, zip, readme)
-      results["pred_training"] = training.evaluate(verbose, llm, zip, readme)
-      results["pred_evaluation"] = validating.evaluate(verbose, llm, zip, readme)
-      results["pred_weights"] = weights.evaluate(verbose, llm, zip, readme)
-      results["pred_readme"] = documentation.evaluate(verbose, llm, zip, readme)
-      results["pred_codetocomment"] = documentation.get_code_to_comment_ratio(zip)
-      pitfalls.evaluate(verbose, llm, zip, readme)
-
-      return results
-  except Exception as e:
-      log(verbose, "ERROR", "Evaluating repository failed: " + str(e))
-      results["pred_live"] = "No"
-      return results
+        results["pred_dependencies"] = requirements.evaluate(verbose, llm, zip, readme)
+        results["pred_training"] = training.evaluate(verbose, llm, zip, readme)
+        results["pred_evaluation"] = validating.evaluate(verbose, llm, zip, readme)
+        results["pred_weights"] = weights.evaluate(verbose, llm, zip, readme)
+        results["pred_readme"] = documentation.evaluate(verbose, llm, zip, readme)
+        results["pred_codetocomment"] = documentation.get_code_to_comment_ratio(zip)
+        pitfalls.evaluate(verbose, llm, zip, readme)
+
+        return results
+    except Exception as e:
+        log(verbose, "ERROR", "Evaluating repository failed: " + str(e))
+        results["pred_live"] = "No"
+        return results
 
 def full_evaluation():
   paper_dump = pd.read_csv("data/dump.csv", sep="\t")
@@ -82,7 +84,7 @@ def full_evaluation():
 def midl_evaluations(model):
   compare_to_gt = True
   paper_dump = pd.read_csv("data/dump.csv", sep="\t")
-  verbose = 1
+  verbose = 0
 
   eval_readme = []
   eval_training = []
@@ -101,7 +103,6 @@ def midl_evaluations(model):
       if (pd.isna(row["url"]) | (row["url"] == "")):
           continue
 
-
       print(f"\nEvaluating {idx+1} out of {len(paper_dump.index)} papers...")
       print(f'Paper title - "{row["title"]}" ({row["year"]})')
       print(f'Repository link - {row["url"]}')
@@ -111,6 +112,7 @@ def midl_evaluations(model):
       full_results.append(row)
       if (compare_to_gt):
           print("\nSummary:")
+          print(row["NA"])
           if ((~pd.isna(row["dependencies"])) & (row["pred_dependencies"] is not None)):
               eval_dependencies.append(row["pred_dependencies"] == row["dependencies"])
               print(f"Dependencies acc. - {row['pred_dependencies']} (GT:{row['dependencies']}) / {int(100 * np.mean(eval_dependencies))}%")

midl.py

@@ -6,7 +6,7 @@ load_dotenv()
 token = os.getenv("githubToken")
 
 # Load model directly
-model = LocalLLM("meta-llama/Llama-3.1-8B-Instruct")
+model = LocalLLM("meta-llama/Llama-3.2-3B-Instruct")
 
 res = midl_evaluations(model)
 res.to_csv("results_midl.csv", sep="\t", index=False)