calculate scores func added

app.py

@@ -8,7 +8,8 @@ from sentence_transformers import SentenceTransformer
 from searchmilvushelper import SearchTopKDocuments
 from finetuneresults import FineTuneAndRerankSearchResults
 from generationhelper import GenerateAnswer
-from calculatescorehelper import CalculateScoresBasedOnAnswer
+from formatresultshelper import CalculateScoresBasedOnAnswer
+from calculatescores import CalculateScores
 
 from model import generate_response
 from huggingface_hub import login
@@ -54,6 +55,12 @@ completion_result = CalculateScoresBasedOnAnswer(query, reranked_results.head(1)
 
 print(completion_result)
 
+score1, score2, score3 = CalculateScores()
+
+print(score1)
+print(score2)
+print(score3)
+
 
 def chatbot(prompt):
     return whoami()

calculatescores.py

@@ -0,0 +1,57 @@
+import formatresultshelper
+
+#Defined as utilized documents / retrieved documents for the query
+def compute_context_relevance(relevant_sentences, support_keys):
+    total_relevance_score = 0
+    total_relevant_sentences = len(relevant_sentences)
+
+    for sentence in relevant_sentences:
+      if sentence in support_keys:
+        total_relevance_score += 1
+
+    # To avoid division by zero in case there are no relevant sentences
+    if total_relevant_sentences == 0:
+        return 0
+
+    return total_relevance_score / total_relevant_sentences
+
+def compute_context_utilization(relevant_sentences, utilization_levels):
+    total_utilization_score = 0
+    total_relevant_sentences = len(relevant_sentences)
+    for sentence in relevant_sentences:
+      if sentence in utilization_levels:
+        total_utilization_score += 1
+    # To avoid division by zero in case there are no relevant sentences
+    if total_relevant_sentences == 0:
+        return 0
+    return total_utilization_score / total_relevant_sentences
+
+
+def CalculateScores():
+   #compute Context Relevance
+   contextrel = compute_context_relevance(formatresultshelper.relevant_sentence_keys, formatresultshelper.support_keys)
+   print(f"Context Relevance = {contextrel}")
+
+   contextutil = compute_context_utilization(formatresultshelper.relevant_sentence_keys, formatresultshelper.all_utilized_sentence_keys)
+   print(f"Context Utilization = {contextutil}")
+
+   compnum = np.intersect1d(formatresultshelper.support_keys, formatresultshelper.all_utilized_sentence_keys)
+   completenes = compnum.size / len(formatresultshelper.support_keys)
+   print(f"Completeness = {completenes}")
+
+   #Adherence : whether all parts of response are grounded by context
+   for val in formatresultshelper.support_level:
+     prevval = 1;
+     if val == False:
+       adherence = 0 * prevval
+       break
+     else:
+       adherence = 1 * prevval
+     prevval = adherence
+
+   print(f"Adherence = {adherence}")
+
+   
+
+def mse(actual, predicted):
+    return (actual - predicted)**2   

calculatescorehelper.py → formatresultshelper.py

@@ -1,4 +1,5 @@
 import generationhelper
+import json
 
 def evaluate_response_with_prompt(templete, query, documents, answer, eval_model="llama-3.3-70b-specdec"):
 
@@ -33,8 +34,51 @@ def CalculateScoresBasedOnAnswer(query, documents, answer, eval_model):
    completion_results = evaluate_response_with_prompt(templete, query,documents, answer, eval_model)
 
    print(completion_results)
+   completion_results_response = completion_results.choices[0].message.content
+   completion_results_response = completion_results_response.strip().strip('```')
+   print(completion_results_response)
 
-   return completion_results
+   # Check if response_content is empty
+   if not completion_results_response.strip():
+      raise ValueError("Empty response content")
+   
+   # Decode if it's a byte string
+   if isinstance(completion_results_response, bytes):
+      completion_results_response = completion_results_response.decode('utf-8')
+
+   # Try to parse JSON
+   try:
+        data_json = json.loads(completion_results_response)
+        print("JSON parsed successfully:")
+        print(data_json)
+   except json.JSONDecodeError as e:
+        print(f"Failed to parse JSON: {e}")
+        print(f"Response content: {completion_results_response}")
+
+   relavance_explanation = data_json['relevance_explanation']
+   relevant_sentence_keys = data_json['all_relevant_sentence_keys']
+   overall_supported_explanation = data_json['overall_supported_explanation']
+   overall_supported = data_json['overall_supported']
+   sentence_support_information = data_json['sentence_support_information']
+   all_utilized_sentence_keys = data_json['all_utilized_sentence_keys']
+
+   print(relavance_explanation)
+   print(relevant_sentence_keys)
+   print(overall_supported_explanation)
+   print(overall_supported)
+   print(sentence_support_information)
+   print(all_utilized_sentence_keys)
+
+   support_keys = []
+   support_level = []
+   for sentence_support in sentence_support_information:
+     support_keys += sentence_support['supporting_sentence_keys']
+     support_level.append(sentence_support['fully_supported'])
+
+   print(support_keys)
+   print(support_level)
+
+   return completion_results_response