app.py

import nltk
import re
import nltkmodule

from newspaper import Article, fulltext
import requests
import itertools
import os

from nltk.tokenize import word_tokenize, sent_tokenize
from sentence_transformers import SentenceTransformer
import pandas as pd
import numpy as np
from pandas import ExcelWriter
from torch.utils.data import DataLoader
import math
from sentence_transformers import models
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.metrics.pairwise import cosine_similarity
import scipy.spatial
import networkx as nx
import scispacy
import spacy
import en_core_sci_lg
import string
from nltk.stem.wordnet import WordNetLemmatizer
import gradio as gr
import inflect
from sklearn.metrics import silhouette_score
from xml.etree import ElementTree as ET

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

p = inflect.engine()
nlp = en_core_sci_lg.load()
sp = en_core_sci_lg.load()
all_stopwords = sp.Defaults.stop_words

os.environ["TOKENIZERS_PARALLELISM"] = "false"

def remove_stopwords(sen):
    sen_new = " ".join([i for i in sen if i not in stop_words])
    return sen_new

# ------------- Evidence Extraction with NLI Model (Global Model Loading) -------------
NLI_MODEL_NAME = "pritamdeka/PubMedBERT-MNLI-MedNLI"
nli_tokenizer = AutoTokenizer.from_pretrained(NLI_MODEL_NAME)
nli_model = AutoModelForSequenceClassification.from_pretrained(NLI_MODEL_NAME)
NLI_LABELS = ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT']  # typical MNLI order

def extract_evidence_sentences(claim, abstracts):
    results = []
    for title, abstract in zip(abstracts['Title'], abstracts['Abstract']):
        sentences = sent_tokenize(abstract)
        evidence = []
        for sent in sentences:
            # premise = sent, hypothesis = claim
            encoding = nli_tokenizer(
                sent, claim,
                return_tensors='pt',
                truncation=True,
                max_length=256,
                padding=True
            )
            with torch.no_grad():
                outputs = nli_model(**encoding)
                probs = torch.softmax(outputs.logits, dim=1).cpu().numpy().flatten()
                max_idx = probs.argmax()
                label = NLI_LABELS[max_idx]
                score = float(probs[max_idx])
            evidence.append({
                "sentence": sent,
                "label": label,
                "score": score
            })
        results.append({
            "title": title,
            "evidence": evidence
        })
    return results

def keyphrase_generator(
    article_link, model_1, model_2, max_num_keywords, model_3, max_retrieved, model_4, extract_evidence):

    # ---------- Robust Article Download ----------
    try:
        response = requests.get(article_link, timeout=20)
        response.raise_for_status()
        html = response.text
        article = fulltext(html)
    except Exception as e:
        return {"error": f"Failed to download article: {str(e)}"}

    corpus = sent_tokenize(article)

    # ---------- TextRank + Keyphrase Extraction ----------
    model_1 = SentenceTransformer(model_1)
    model_2 = SentenceTransformer(model_2)

    indicator_list = ['concluded', 'concludes', 'in a study', 'concluding', 'conclude', 'in sum', 'in a recent study',
                      'therefore', 'thus', 'so', 'hence', 'as a result', 'accordingly', 'consequently', 'in short',
                      'proves that', 'shows that', 'suggests that', 'demonstrates that', 'found that', 'observed that',
                      'indicated that', 'suggested that', 'demonstrated that']
    score_list = []
    count_dict = {}
    for l in corpus:
        c = 0
        for l2 in indicator_list:
            if l.find(l2) != -1:
                c = 1
                break
        count_dict[l] = c
    for sent, score in count_dict.items():
        score_list.append(score)
    clean_sentences_new = pd.Series(corpus).str.replace("[^a-zA-Z]", " ", regex=True).tolist()
    corpus_embeddings = model_1.encode(clean_sentences_new)
    sim_mat = np.zeros([len(clean_sentences_new), len(clean_sentences_new)])
    for i in range(len(clean_sentences_new)):
        len_embeddings = len(corpus_embeddings[i])
        for j in range(len(clean_sentences_new)):
            if i != j:
                if len_embeddings == 1024:
                    sim_mat[i][j] = cosine_similarity(corpus_embeddings[i].reshape(1, 1024),
                                                      corpus_embeddings[j].reshape(1, 1024))[0, 0]
                elif len_embeddings == 768:
                    sim_mat[i][j] = cosine_similarity(corpus_embeddings[i].reshape(1, 768),
                                                      corpus_embeddings[j].reshape(1, 768))[0, 0]
    nx_graph = nx.from_numpy_array(sim_mat)
    scores = nx.pagerank(nx_graph, max_iter=1500)
    sentences = ((scores[i], s) for i, s in enumerate(corpus))
    element = [elem[0] for elem in sentences]
    sum_list = [sc + lst for sc, lst in zip(score_list, element)]
    x = sorted(((sum_list[i], s) for i, s in enumerate(corpus)), reverse=True)
    final_textrank_list = [elem[1] for elem in x]
    a = int((10 * len(final_textrank_list)) / 100.0)
    total = max(a, 5)
    document = [final_textrank_list[i] for i in range(total)]
    doc = " ".join(document)
    text_doc = []
    for i in document:
        doc_1 = nlp(i)
        text_doc.append([X.text for X in doc_1.ents])
    entity_list = [item for sublist in text_doc for item in sublist]
    entity_list = [word for word in entity_list if word not in all_stopwords]
    entity_list = [word_entity for word_entity in entity_list if not p.singular_noun(word_entity)]
    entity_list = list(dict.fromkeys(entity_list))
    doc_embedding = model_2.encode([doc])
    candidates = entity_list
    candidate_embeddings = model_2.encode(candidates)
    distances = cosine_similarity(doc_embedding, candidate_embeddings)
    top_n = max_num_keywords
    keyword_list = [candidates[index] for index in distances.argsort()[0][-top_n:]]
    # ---------- Clustering + Query Generation ----------
    word_embedding_model = models.Transformer(model_3)
    pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
                                  pooling_mode_mean_tokens=True,
                                  pooling_mode_cls_token=False,
                                  pooling_mode_max_tokens=False)
    embedder = SentenceTransformer(modules=[word_embedding_model, pooling_model])
    c_len = len(keyword_list)
    keyword_embeddings = embedder.encode(keyword_list)
    silhouette_score_list = []
    cluster_list_final = []
    for num_clusters in range(1, top_n):
        clustering_model = KMeans(n_clusters=num_clusters)
        clustering_model.fit(keyword_embeddings)
        cluster_assignment = clustering_model.labels_
        clustered_sentences = [[] for _ in range(num_clusters)]
        for sentence_id, cluster_id in enumerate(cluster_assignment):
            clustered_sentences[cluster_id].append(keyword_list[sentence_id])
        cl_sent_len = len(clustered_sentences)
        list_cluster = list(clustered_sentences)
        cluster_list_final.append(list_cluster)
        if (c_len == cl_sent_len and c_len >= 3) or cl_sent_len == 1:
            silhouette_avg = 0
        elif c_len == cl_sent_len == 2:
            silhouette_avg = 1
        else:
            silhouette_avg = silhouette_score(keyword_embeddings, cluster_assignment)
        silhouette_score_list.append(silhouette_avg)
    res_dict = dict(zip(silhouette_score_list, cluster_list_final))
    cluster_items = res_dict[max(res_dict)]
    comb = []
    for i in cluster_items:
        z = ' OR '.join(i)
        comb.append("(" + z + ")")
    combinations = []
    for subset in itertools.combinations(comb, 2):
        combinations.append(subset)
    f1_list = []
    for s in combinations:
        final = ' AND '.join(s)
        f1_list.append("(" + final + ")")
    f_1 = ' OR '.join(f1_list)
    # ---------- PubMed Abstract Extraction ----------
    ncbi_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
    last_url = 'esearch.fcgi?db=pubmed' + '&term=' + f_1
    overall_url = ncbi_url + last_url + '&rettype=json' + '&sort=relevance'
    try:
        pubmed_search_request = requests.get(overall_url, timeout=20)
        root = ET.fromstring(pubmed_search_request.text)
        levels = root.findall('.//Id')
        search_id_list = [level.text for level in levels]
        if not search_id_list:
            return {"error": "No PubMed results found."}
    except Exception as e:
        return {"error": f"Error retrieving from PubMed: {str(e)}"}
    all_search_ids = ','.join(search_id_list)
    fetch_url = 'efetch.fcgi?db=pubmed'
    search_id = '&id=' + all_search_ids
    return_url = ncbi_url + fetch_url + search_id + '&rettype=text' + '&retmode=xml' + '&retmax=500' + '&sort=relevance'
    try:
        pubmed_abstract_request = requests.get(return_url, timeout=20)
        root_1 = ET.fromstring(pubmed_abstract_request.text)
        article_title = root_1.findall('.//ArticleTitle')
        titles_list = [a.text for a in article_title]
        article_abstract = root_1.findall('.//AbstractText')
        abstracts_list = [b.text for b in article_abstract]
    except Exception as e:
        return {"error": f"Error extracting PubMed abstracts: {str(e)}"}
    if not titles_list or not abstracts_list:
        return {"error": "No abstracts found for this query."}
    # ---------- Most relevant abstracts by heading ----------
    try:
        first_article = Article(article_link, language='en')
        first_article.download()
        first_article.parse()
        article_heading = first_article.title
        if not article_heading or not isinstance(article_heading, str):
            article_heading = corpus[0] if corpus else ""
    except Exception:
        article_heading = corpus[0] if corpus else ""
    model_4 = SentenceTransformer(model_4)
    my_dict = dict(zip(titles_list, abstracts_list))
    title_embeddings = model_4.encode(titles_list)
    heading_embedding = model_4.encode([article_heading])
    similarities = cosine_similarity(heading_embedding, title_embeddings)
    max_n = max_retrieved
    sorted_titles = [titles_list[index] for index in similarities.argsort()[0][-max_n:]]
    sorted_abstract_list = [my_dict[list_elem] for list_elem in sorted_titles]
    sorted_dict = {'Title': sorted_titles, 'Abstract': sorted_abstract_list}

    # ---------- Evidence Extraction Integration ----------
    if extract_evidence:
        evidence_results = extract_evidence_sentences(
            article_heading,
            sorted_dict,
        )
        return evidence_results
    else:
        return sorted_dict

igen_pubmed = gr.Interface(
    keyphrase_generator,
    inputs=[
        gr.components.Textbox(lines=1, placeholder="Provide article web link here", value="", label="Article web link"),
        gr.components.Dropdown(
            choices=[
                'sentence-transformers/all-mpnet-base-v2',
                'sentence-transformers/all-mpnet-base-v1',
                'sentence-transformers/all-distilroberta-v1',
                'sentence-transformers/gtr-t5-large',
                'pritamdeka/S-Bluebert-snli-multinli-stsb',
                'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb',
                'pritamdeka/S-BioBert-snli-multinli-stsb',
                'sentence-transformers/stsb-mpnet-base-v2',
                'sentence-transformers/stsb-roberta-base-v2',
                'sentence-transformers/stsb-distilroberta-base-v2',
                'sentence-transformers/sentence-t5-large',
                'sentence-transformers/sentence-t5-base'
            ],
            type="value",
            value='sentence-transformers/stsb-roberta-base-v2',
            label="Select any SBERT model for TextRank"
        ),
        gr.components.Dropdown(
            choices=[
                'sentence-transformers/paraphrase-mpnet-base-v2',
                'sentence-transformers/all-mpnet-base-v1',
                'sentence-transformers/paraphrase-distilroberta-base-v1',
                'sentence-transformers/paraphrase-xlm-r-multilingual-v1',
                'sentence-transformers/paraphrase-multilingual-mpnet-base-v2',
                'sentence-transformers/paraphrase-albert-small-v2',
                'sentence-transformers/paraphrase-albert-base-v2',
                'sentence-transformers/paraphrase-MiniLM-L12-v2',
                'sentence-transformers/paraphrase-MiniLM-L6-v2',
                'sentence-transformers/all-MiniLM-L12-v2',
                'sentence-transformers/all-distilroberta-v1',
                'sentence-transformers/paraphrase-TinyBERT-L6-v2',
                'sentence-transformers/paraphrase-MiniLM-L3-v2',
                'sentence-transformers/all-MiniLM-L6-v2'
            ],
            type="value",
            value='sentence-transformers/all-mpnet-base-v1',
            label="Select any SBERT model for keyphrases"
        ),
        gr.components.Slider(minimum=5, maximum=20, step=1, value=10, label="Max Keywords"),
        gr.components.Dropdown(
            choices=[
                'cambridgeltl/SapBERT-from-PubMedBERT-fulltext',
                'cambridgeltl/SapBERT-from-PubMedBERT-fulltext-mean-token'
            ],
            type="value",
            value='cambridgeltl/SapBERT-from-PubMedBERT-fulltext',
            label="Select any SapBERT model for clustering"
        ),
        gr.components.Slider(minimum=5, maximum=15, step=1, value=10, label="PubMed Max Abstracts"),
        gr.components.Dropdown(
            choices=[
                'pritamdeka/S-Bluebert-snli-multinli-stsb',
                'pritamdeka/S-BioBert-snli-multinli-stsb',
                'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb',
                'sentence-transformers/all-mpnet-base-v2'
            ],
            type="value",
            value='sentence-transformers/all-mpnet-base-v2',
            label="Select any SBERT model for abstracts"
        ),
        gr.components.Checkbox(label="Enable Evidence Extraction", value=True)
    ],
    outputs=gr.components.JSON(label="Results (Abstracts + Evidence)"),
    title="PubMed Abstract Retriever",
    description="Retrieves relevant PubMed abstracts for an online article and optionally extracts evidence for the claim made in the article headline. Outputs JSON mapping each abstract to evidence sentences and their stance (ENTAILMENT/SUPPORT, CONTRADICTION, NEUTRAL).",
    examples=[
        [
            "https://www.cancer.news/2021-12-22-mrna-vaccines-weaken-immune-system-cause-cancer.html",
            'sentence-transformers/all-mpnet-base-v1',
            'sentence-transformers/paraphrase-MiniLM-L12-v2',
            10,
            'cambridgeltl/SapBERT-from-PubMedBERT-fulltext',
            15,
            'pritamdeka/S-Biomed-Roberta-snli-multinli-stsb',
            True
        ]
    ]
)

igen_pubmed.launch(share=False, server_name='0.0.0.0', show_error=True)