Refactor paper summarization to improve text handling and remove unused parameters

main.py

@@ -10,6 +10,16 @@ from send_mail import send_email
 dotenv.load_dotenv()
 access_key = os.getenv("ACCESS_KEY")
 
+def fix_text(text):
+    text = html.escape(text.encode('utf-8').decode('utf-8').replace("â¦", "..., "))
+    fixed_text = ""
+    for word in text.split():
+        try:
+            fixed_text += word.encode('latin1').decode('utf-8')+" "
+        except:
+            fixed_text += word+" "
+    return fixed_text.encode('utf-8').decode().replace('\\', '\\\\')
+
 def paper_data(paper_data, wait_time=5):
     data = {"status": "success"}
     data['data'] = {}
@@ -21,23 +31,20 @@ def paper_data(paper_data, wait_time=5):
                 doi = details.get("doi")
                 pdf_url = details.get("pdf_url")
                 title = details.get("title")
+                title = html.escape(title.encode('unicode-escape').decode().replace('\\\\', '\\'))
                 citation = details.get("citation")
                 if not all([paper_id, doi, pdf_url, title, citation]):
                     print(f"Skipping paper with ID: {paper_id} (missing details)")
                     continue
-                fixed_title, summary, mindmap, fixed_citation = summarize_paper(title, pdf_url, paper_id, citation, access_key)
-                if not fixed_title:
-                    title = title
-                else:
-                    title = fixed_title
-                if not fixed_citation:
-                    citation = citation
-                else:
-                    citation = fixed_citation
+                summary, mindmap = summarize_paper(pdf_url, paper_id, access_key)
                 if not summary or not mindmap:
                     print(f"Skipping paper with ID: {paper_id} (Summary/Mindmap not found)")
                     continue
                 try:
+                    title = fix_text(title)
+                    summary = fix_text(summary)
+                    mindmap = fix_text(mindmap)
+                    citation = fix_text(citation)
                     title = html.escape(str(title).strip())
                     citation = html.escape(str(citation).strip())
                     status = post_blog(doi, title, category, summary, mindmap, citation, access_key, wait_time)

summarize_paper.py

@@ -1,32 +1,28 @@
+import os
 import json
 from gradio_client import Client
 
-def summarize_paper(paper_title, pdf_url, paper_id, paper_citation, access_key):
+def summarize_paper(pdf_url, paper_id, access_key):
     mindmap = None
     summary = None
-    title = None
-    citation = None
     try:
-        summarizer_client = Client("raannakasturi/ReXploreAPI")
+        summarizer_client = Client(
+            "raannakasturi/ReXploreAPI",
+            hf_token=os.environ.get("HF_API_TOKEN"),
+        )
         result = summarizer_client.predict(
                 url=pdf_url,
-                title=paper_title,
                 id=paper_id,
-                citation=paper_citation,
                 access_key=access_key,
                 api_name="/rexplore_summarizer"
         )
         if result:
             data = json.loads(result[0])
             print
-            if data['title']:
-                title = data['title']
-            if data['citation']:
-                citation = data['citation']
             if data["mindmap_status"] == "success":
                 mindmap = data["mindmap"]
             if data["summary_status"] == "success":
                 summary = data["summary"]
     except Exception as e:
         print(f"Error summarizing paper: {e}")
-    return title, summary, mindmap, citation
+    return summary, mindmap