data/fetch_processed.py

import csv
import numpy as np
import pandas as pd
import re

current_year = 2024
MIDL_years = range(2018, current_year + 1, 1)
custom_order = ["MICCAI", "MIDL", "Nature", "arXiv"]

for venue in custom_order:
    df = pd.read_excel("https://docs.google.com/spreadsheets/d/e/2PACX-1vQjpsSYcEcYUVB-88bCQ01UfQf0z9m16ax7p1ft03G68Nr-DdXHpPt-xOFSrXFj1N49AjK5nYhmKBfo/pub?output=xlsx", sheet_name=venue)
    df.to_csv(f'data/{venue}.csv', sep="\t")

# Store all evaluations here
paper_dump = pd.DataFrame()
# Official color codes for conferences
MIDL_colors = ["#506775", "#4E7268", "#5170B1", "#004B5A", "#268BCC", "#B18630", "#AA0000"]

for venue in custom_order:
    with open(f'data/{venue}.csv') as file:
        tsv_file = csv.reader(file, delimiter="\t")
        for row in tsv_file:
            if (row[0] == ""):
                continue

            if (row[1] == ""):
                continue

            paper_dump = pd.concat([paper_dump, pd.DataFrame({"venue": venue, "title": [row[1]], "year": [row[2]], "pdf": [row[3]], "url": [row[4]], "public": [row[5]], "dependencies": [row[6]], "training": [row[7]], "evaluation": [row[8]], "weights": [row[9]], "readme": [row[10]], "license": [row[11]]})], ignore_index=True)
paper_dump.to_csv(f'data/dump.csv', sep="\t")