Refactor title and citation encoding in paper_data and test functions for improved handling of special characters

main.py

@@ -30,7 +30,16 @@ def paper_data(paper_data, wait_time=5):
                     print(f"Skipping paper with ID: {paper_id} (summary/mindmap not found)")
                     continue
                 try:
-                    title = html.escape(title.encode('unicode-escape').decode().replace('\\\\', '\\'))
+                    try:
+                        escaped_title = repr(title.encode('latin1').decode('unicode-escape')).strip()
+                    except:
+                        escaped_title = repr(title).strip()
+                    title = html.escape(str(escaped_title).strip()[1:-1])
+                    try:
+                        encoded_bytes = citation.encode('latin1').decode('utf-8', errors='replace')
+                    except:
+                        encoded_bytes = repr(citation)
+                    citation = html.unescape(encoded_bytes)
                     post_blog(title, category, summary, mindmap, citation, access_key, wait_time)
                 except Exception as e:
                     print(f"Error posting blog '{title}': {e}")

post_blog.py

@@ -153,7 +153,7 @@ def test(uaccess_key):
                 "2412.16344": {
                     "id": "2412.20276",
                     "doi": "https://doi.org/10.48550/arXiv.2412.20276",
-                    "title": "Demographics of black holes at $<$100 R$_{\\rm g}$ scales: accretion flows, jets, and shadows",
+                    "title": "From Painlev\xe9 equations to ${\cal N}=2$ susy gauge theories: prolegomena",
                     "category": "Astrophysics",
                     "citation": "Nair, D. G., Nagar, N. M., Ramakrishnan, V., Wielgus, M., Arratia, V., Krichbaum, T. P., Zhang, X. A., Ricarte, A., S., S., Hernández-Yévenes, J., Ford, N. M., Bandyopadhyay, B., Gurwell, M., Burridge, R., Pesce, D. W., Doeleman, S. S., Kim, J.-Y., Kim, D., Janssen, M., … Zensus, J. A. (2024). Demographics of black holes at $<$100 R$_{\\rm g}$ scales: accretion flows, jets, and shadows (Version 1). arXiv. https://doi.org/10.48550/ARXIV.2412.20276",
                     "summary": "## Summary\nThe Event Horizon Telescope (EHT) has imaged supermassive black holes (SMBHs) in M87 and Sgr A*, testing General Relativity and constraining black hole mass. The EHT has also imaged inner jets of several SMBHs, but at coarser resolutions than 100 Rg. The \"Event Horizon and Environs\" (ETHER) sample is a comprehensive database of SMBH mass estimates, radio to hard X-ray flux densities, and spectral energy distribution information.\n\n## Highlights\n- The EHT has imaged SMBH shadows in M87 and Sgr A*, testing General Relativity and constraining black hole mass.\n- The EHT has imaged the inner jets of several SMBHs, but at coarser resolutions than 100 Rg.\n- The ETHER sample is a comprehensive database of SMBH mass estimates, radio to hard X-ray flux densities, and spectral energy distribution information.\n- The ETHER database includes predictions for the 230 GHz and 345 GHz flux densities in the EHT field of view for approximately 40,000 SMBHs.\n- The 43 GHz VLBA is the highest frequency sensitive phase-referenced VLBI network, providing valuable constraints for SED modeling.\n- The ngEHT is expected to measure approximately 50 SMBH masses, 30 SMBH spins, and 7 SMBH shadows.\n- The ngEHT will enable demographic studies of SMBHs, testing General Relativity and constraining black hole mass.\n\n## Key Insights\n- The EHT's ability to image SMBH shadows and inner jets has opened a new window into understanding these enigmatic objects, allowing for tests of General Relativity and constraints on black hole mass.\n- The ETHER sample provides a comprehensive database for identifying and optimizing target selection for the EHT and its upgrades, enabling the identification of a \"Gold Sample\" of nearby low-luminosity Active Galactic Nuclei (AGNs) optimal for studying jet bases and potentially imaging black hole shadows.\n- The ngEHT will enable the measurement of approximately 50 SMBH masses, 30 SMBH spins, and 7 SMBH shadows, providing a significant increase in the number of SMBHs that can be studied in detail.\n- The 43 GHz VLBA provides valuable constraints for SED modeling, allowing for the prediction of EHT flux densities and the identification of optimal targets for EHT observations.\n- The ETHER database includes predictions for the 230 GHz and 345 GHz flux densities in the EHT field of view for approximately 40,000 SMBHs, enabling the identification of potential targets for EHT observations.\n- The ngEHT will enable demographic studies of SMBHs, testing General Relativity and constraining black hole mass, and providing insights into the properties of these enigmatic objects.\n- The EHT's ability to image SMBH shadows and inner jets has significant implications for our understanding of these objects and the role they play in the universe, and will continue to be an important area of research in the coming years.",
@@ -167,12 +167,22 @@ def test(uaccess_key):
         for category, catdata in data.items():
             if category != 'status':
                 for paper_id, paperdata in catdata.items():
-                    title = html.escape(paperdata['title'].encode('unicode-escape').decode().replace('\\\\', '\\'))
+                    title = paperdata['title']
                     category = paperdata['category']
                     summary = paperdata.get('summary', '')
                     mindmap = paperdata.get('mindmap', '')
                     citation = paperdata.get('citation', '')
                     uaccess_key = access_key
+                    try:
+                        escaped_title = repr(title.encode('latin1').decode('unicode-escape')).strip()
+                    except:
+                        escaped_title = repr(title).strip()
+                    title = html.escape(str(escaped_title).strip()[1:-1])
+                    try:
+                        encoded_bytes = citation.encode('latin1').decode('utf-8', errors='replace')
+                    except:
+                        encoded_bytes = repr(citation)
+                    citation = html.unescape(encoded_bytes)
                     status = post_blog(title, category, summary, mindmap, citation, uaccess_key, 0)
                     print(status)
                     return status