import nltk import re import nltkmodule from newspaper import Article, fulltext import requests import itertools import os from nltk.tokenize import word_tokenize, sent_tokenize from sentence_transformers import SentenceTransformer import pandas as pd import numpy as np from pandas import ExcelWriter from torch.utils.data import DataLoader import math from sentence_transformers import models from nltk.corpus import stopwords stop_words = stopwords.words('english') import matplotlib.pyplot as plt from sklearn.cluster import KMeans from sklearn.decomposition import PCA from sklearn.metrics.pairwise import cosine_similarity import scipy.spatial import networkx as nx import scispacy import spacy import en_core_sci_lg import string from nltk.stem.wordnet import WordNetLemmatizer import gradio as gr import inflect from sklearn.metrics import silhouette_score from xml.etree import ElementTree as ET from transformers import AutoTokenizer, AutoModelForSequenceClassification import torch p = inflect.engine() nlp = en_core_sci_lg.load() sp = en_core_sci_lg.load() all_stopwords = sp.Defaults.stop_words os.environ["TOKENIZERS_PARALLELISM"] = "false" def remove_stopwords(sen): sen_new = " ".join([i for i in sen if i not in stop_words]) return sen_new # ------------- Evidence Extraction with NLI Model (Global Model Loading) ------------- NLI_MODEL_NAME = "pritamdeka/PubMedBERT-MNLI-MedNLI" nli_tokenizer = AutoTokenizer.from_pretrained(NLI_MODEL_NAME) nli_model = AutoModelForSequenceClassification.from_pretrained(NLI_MODEL_NAME) NLI_LABELS = ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT'] # typical MNLI order def extract_evidence_sentences(claim, abstracts): results = [] for title, abstract in zip(abstracts['Title'], abstracts['Abstract']): sentences = sent_tokenize(abstract) evidence = [] for sent in sentences: # premise = sent, hypothesis = claim encoding = nli_tokenizer( sent, claim, return_tensors='pt', truncation=True, max_length=256, padding=True ) with torch.no_grad(): outputs = nli_model(**encoding) probs = torch.softmax(outputs.logits, dim=1).cpu().numpy().flatten() max_idx = probs.argmax() label = NLI_LABELS[max_idx] score = float(probs[max_idx]) evidence.append({ "sentence": sent, "label": label, "score": score }) results.append({ "title": title, "evidence": evidence }) return results def keyphrase_generator( article_link, model_1, model_2, max_num_keywords, model_3, max_retrieved, model_4, extract_evidence): # ---------- Robust Article Download ---------- try: response = requests.get(article_link, timeout=20) response.raise_for_status() html = response.text article = fulltext(html) except Exception as e: return {"error": f"Failed to download article: {str(e)}"} corpus = sent_tokenize(article) # ---------- TextRank + Keyphrase Extraction ---------- model_1 = SentenceTransformer(model_1) model_2 = SentenceTransformer(model_2) indicator_list = ['concluded', 'concludes', 'in a study', 'concluding', 'conclude', 'in sum', 'in a recent study', 'therefore', 'thus', 'so', 'hence', 'as a result', 'accordingly', 'consequently', 'in short', 'proves that', 'shows that', 'suggests that', 'demonstrates that', 'found that', 'observed that', 'indicated that', 'suggested that', 'demonstrated that'] score_list = [] count_dict = {} for l in corpus: c = 0 for l2 in indicator_list: if l.find(l2) != -1: c = 1 break count_dict[l] = c for sent, score in count_dict.items(): score_list.append(score) clean_sentences_new = pd.Series(corpus).str.replace("[^a-zA-Z]", " ", regex=True).tolist() corpus_embeddings = model_1.encode(clean_sentences_new) sim_mat = np.zeros([len(clean_sentences_new), len(clean_sentences_new)]) for i in range(len(clean_sentences_new)): len_embeddings = len(corpus_embeddings[i]) for j in range(len(clean_sentences_new)): if i != j: if len_embeddings == 1024: sim_mat[i][j] = cosine_similarity(corpus_embeddings[i].reshape(1, 1024), corpus_embeddings[j].reshape(1, 1024))[0, 0] elif len_embeddings == 768: sim_mat[i][j] = cosine_similarity(corpus_embeddings[i].reshape(1, 768), corpus_embeddings[j].reshape(1, 768))[0, 0] nx_graph = nx.from_numpy_array(sim_mat) scores = nx.pagerank(nx_graph, max_iter=1500) sentences = ((scores[i], s) for i, s in enumerate(corpus)) element = [elem[0] for elem in sentences] sum_list = [sc + lst for sc, lst in zip(score_list, element)] x = sorted(((sum_list[i], s) for i, s in enumerate(corpus)), reverse=True) final_textrank_list = [elem[1] for elem in x] a = int((10 * len(final_textrank_list)) / 100.0) total = max(a, 5) document = [final_textrank_list[i] for i in range(total)] doc = " ".join(document) text_doc = [] for i in document: doc_1 = nlp(i) text_doc.append([X.text for X in doc_1.ents]) entity_list = [item for sublist in text_doc for item in sublist] entity_list = [word for word in entity_list if word not in all_stopwords] entity_list = [word_entity for word_entity in entity_list if not p.singular_noun(word_entity)] entity_list = list(dict.fromkeys(entity_list)) doc_embedding = model_2.encode([doc]) candidates = entity_list candidate_embeddings = model_2.encode(candidates) distances = cosine_similarity(doc_embedding, candidate_embeddings) top_n = max_num_keywords keyword_list = [candidates[index] for index in distances.argsort()[0][-top_n:]] # ---------- Clustering + Query Generation ---------- word_embedding_model = models.Transformer(model_3) pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), pooling_mode_mean_tokens=True, pooling_mode_cls_token=False, pooling_mode_max_tokens=False) embedder = SentenceTransformer(modules=[word_embedding_model, pooling_model]) c_len = len(keyword_list) keyword_embeddings = embedder.encode(keyword_list) silhouette_score_list = [] cluster_list_final = [] for num_clusters in range(1, top_n): clustering_model = KMeans(n_clusters=num_clusters) clustering_model.fit(keyword_embeddings) cluster_assignment = clustering_model.labels_ clustered_sentences = [[] for _ in range(num_clusters)] for sentence_id, cluster_id in enumerate(cluster_assignment): clustered_sentences[cluster_id].append(keyword_list[sentence_id]) cl_sent_len = len(clustered_sentences) list_cluster = list(clustered_sentences) cluster_list_final.append(list_cluster) if (c_len == cl_sent_len and c_len >= 3) or cl_sent_len == 1: silhouette_avg = 0 elif c_len == cl_sent_len == 2: silhouette_avg = 1 else: silhouette_avg = silhouette_score(keyword_embeddings, cluster_assignment) silhouette_score_list.append(silhouette_avg) res_dict = dict(zip(silhouette_score_list, cluster_list_final)) cluster_items = res_dict[max(res_dict)] comb = [] for i in cluster_items: z = ' OR '.join(i) comb.append("(" + z + ")") combinations = [] for subset in itertools.combinations(comb, 2): combinations.append(subset) f1_list = [] for s in combinations: final = ' AND '.join(s) f1_list.append("(" + final + ")") f_1 = ' OR '.join(f1_list) # ---------- PubMed Abstract Extraction ---------- ncbi_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/' last_url = 'esearch.fcgi?db=pubmed' + '&term=' + f_1 overall_url = ncbi_url + last_url + '&rettype=json' + '&sort=relevance' try: pubmed_search_request = requests.get(overall_url, timeout=20) root = ET.fromstring(pubmed_search_request.text) levels = root.findall('.//Id') search_id_list = [level.text for level in levels] if not search_id_list: return {"error": "No PubMed results found."} except Exception as e: return {"error": f"Error retrieving from PubMed: {str(e)}"} all_search_ids = ','.join(search_id_list) fetch_url = 'efetch.fcgi?db=pubmed' search_id = '&id=' + all_search_ids return_url = ncbi_url + fetch_url + search_id + '&rettype=text' + '&retmode=xml' + '&retmax=500' + '&sort=relevance' try: pubmed_abstract_request = requests.get(return_url, timeout=20) root_1 = ET.fromstring(pubmed_abstract_request.text) article_title = root_1.findall('.//ArticleTitle') titles_list = [a.text for a in article_title] article_abstract = root_1.findall('.//AbstractText') abstracts_list = [b.text for b in article_abstract] except Exception as e: return {"error": f"Error extracting PubMed abstracts: {str(e)}"} if not titles_list or not abstracts_list: return {"error": "No abstracts found for this query."} # ---------- Most relevant abstracts by heading ---------- try: first_article = Article(article_link, language='en') first_article.download() first_article.parse() article_heading = first_article.title if not article_heading or not isinstance(article_heading, str): article_heading = corpus[0] if corpus else "" except Exception: article_heading = corpus[0] if corpus else "" model_4 = SentenceTransformer(model_4) my_dict = dict(zip(titles_list, abstracts_list)) title_embeddings = model_4.encode(titles_list) heading_embedding = model_4.encode([article_heading]) similarities = cosine_similarity(heading_embedding, title_embeddings) max_n = max_retrieved sorted_titles = [titles_list[index] for index in similarities.argsort()[0][-max_n:]] sorted_abstract_list = [my_dict[list_elem] for list_elem in sorted_titles] sorted_dict = {'Title': sorted_titles, 'Abstract': sorted_abstract_list} # ---------- Evidence Extraction Integration ---------- if extract_evidence: evidence_results = extract_evidence_sentences( article_heading, sorted_dict, ) return evidence_results else: return sorted_dict igen_pubmed = gr.Interface( keyphrase_generator, inputs=[ gr.components.Textbox(lines=1, placeholder="Provide article web link here", value="", label="Article web link"), gr.components.Dropdown( choices=[ 'sentence-transformers/all-mpnet-base-v2', 'sentence-transformers/all-mpnet-base-v1', 'sentence-transformers/all-distilroberta-v1', 'sentence-transformers/gtr-t5-large', 'pritamdeka/S-Bluebert-snli-multinli-stsb', 'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb', 'pritamdeka/S-BioBert-snli-multinli-stsb', 'sentence-transformers/stsb-mpnet-base-v2', 'sentence-transformers/stsb-roberta-base-v2', 'sentence-transformers/stsb-distilroberta-base-v2', 'sentence-transformers/sentence-t5-large', 'sentence-transformers/sentence-t5-base' ], type="value", value='sentence-transformers/stsb-roberta-base-v2', label="Select any SBERT model for TextRank" ), gr.components.Dropdown( choices=[ 'sentence-transformers/paraphrase-mpnet-base-v2', 'sentence-transformers/all-mpnet-base-v1', 'sentence-transformers/paraphrase-distilroberta-base-v1', 'sentence-transformers/paraphrase-xlm-r-multilingual-v1', 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2', 'sentence-transformers/paraphrase-albert-small-v2', 'sentence-transformers/paraphrase-albert-base-v2', 'sentence-transformers/paraphrase-MiniLM-L12-v2', 'sentence-transformers/paraphrase-MiniLM-L6-v2', 'sentence-transformers/all-MiniLM-L12-v2', 'sentence-transformers/all-distilroberta-v1', 'sentence-transformers/paraphrase-TinyBERT-L6-v2', 'sentence-transformers/paraphrase-MiniLM-L3-v2', 'sentence-transformers/all-MiniLM-L6-v2' ], type="value", value='sentence-transformers/all-mpnet-base-v1', label="Select any SBERT model for keyphrases" ), gr.components.Slider(minimum=5, maximum=20, step=1, value=10, label="Max Keywords"), gr.components.Dropdown( choices=[ 'cambridgeltl/SapBERT-from-PubMedBERT-fulltext', 'cambridgeltl/SapBERT-from-PubMedBERT-fulltext-mean-token' ], type="value", value='cambridgeltl/SapBERT-from-PubMedBERT-fulltext', label="Select any SapBERT model for clustering" ), gr.components.Slider(minimum=5, maximum=15, step=1, value=10, label="PubMed Max Abstracts"), gr.components.Dropdown( choices=[ 'pritamdeka/S-Bluebert-snli-multinli-stsb', 'pritamdeka/S-BioBert-snli-multinli-stsb', 'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb', 'sentence-transformers/all-mpnet-base-v2' ], type="value", value='sentence-transformers/all-mpnet-base-v2', label="Select any SBERT model for abstracts" ), gr.components.Checkbox(label="Enable Evidence Extraction", value=True) ], outputs=gr.components.JSON(label="Results (Abstracts + Evidence)"), title="PubMed Abstract Retriever", description="Retrieves relevant PubMed abstracts for an online article and optionally extracts evidence for the claim made in the article headline. Outputs JSON mapping each abstract to evidence sentences and their stance (ENTAILMENT/SUPPORT, CONTRADICTION, NEUTRAL).", examples=[ [ "https://www.cancer.news/2021-12-22-mrna-vaccines-weaken-immune-system-cause-cancer.html", 'sentence-transformers/all-mpnet-base-v1', 'sentence-transformers/paraphrase-MiniLM-L12-v2', 10, 'cambridgeltl/SapBERT-from-PubMedBERT-fulltext', 15, 'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb', True ] ] ) igen_pubmed.launch(share=False, server_name='0.0.0.0', show_error=True)