Upload 4 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index_bo_tp_cosine_full2.idx filter=lfs diff=lfs merge=lfs -text
+faiss_metadata_bo_tp_full2.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,471 @@
+import gradio as gr
+import torch
+from transformers import TextStreamer
+import numpy as np
+import faiss
+import json
+import re
+from unsloth import FastLanguageModel
+from serpapi import GoogleSearch
+from sentence_transformers import CrossEncoder
+from sentence_transformers import SentenceTransformer
+print("Load QA model")
+new_model_name = "thailevann/Qwen3-4B_SFT_dvcqg_v4"
+model1, tokenizer1 = FastLanguageModel.from_pretrained(
+    model_name = new_model_name,
+    max_seq_length = 2048,
+    dtype = None,
+    load_in_4bit = True,
+)
+FastLanguageModel.for_inference(model1);
+
+
+print("Load embedding model")
+model_embed = SentenceTransformer("huyydangg/DEk21_hcmute_embedding")
+
+print("Load rerank model")
+rerank_model = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+index = faiss.read_index("./faiss_index_bo_tp_cosine_full2.idx")
+with open("./faiss_metadata_bo_tp_full2.json", "r", encoding="utf-8") as f:
+    metadata = json.load(f)
+import re
+import numpy as np
+from bs4 import BeautifulSoup
+import requests
+import json
+from datasets import load_dataset
+from google import genai
+import json
+import time
+
+# Cấu hình API key
+GEMINI_API_KEY = "AIzaSyDi-ZXE3qXfDdIjeDFS8Cw-GylXD4P4LsQ"
+client_gemini = genai.Client(api_key=GEMINI_API_KEY)
+
+
+# Tạo prompt
+def prompt1(question, relevant) -> str:
+    return f"""
+    Bạn là một trợ lý AI hỗ trợ tra cứu dịch vụ công dựa vào thông tin liên quan
+    ## Câu hỏi:
+    {question}
+    
+    ## Thông tin liên quan:
+    { relevant}
+    """
+
+# Hàm gọi Gemini
+def generate_mcq(question, relevant):
+    prompt = prompt1(question, relevant)
+
+    response = client_gemini.models.generate_content(
+        model="gemini-2.0-flash",
+        contents=prompt
+    )
+
+    return response.text
+def generate(q):
+    prompt = f"""
+    Viết lại cho dễ hiểu. Chỉ trả lời, nếu bạn thấy có thông tin không hợp lí có thể sửa lại và không cần giải thích. Các link đều được quy về https://dichvucong.gov.vn/
+
+    {q}
+    """
+    try:
+        response = client_gemini.models.generate_content(
+            model="gemini-2.0-flash",
+            contents=prompt
+        )
+        print(q)
+        return response.text
+    except Exception as e:
+        return q
+
+def extract_table_to_kv(table_tag):
+    headers = [
+        th.get_text(strip=True).lower()
+        .replace(" ", "_")
+        .replace(":", "")
+        for th in table_tag.find_all("th")
+    ]
+
+    rows = []
+    for tr in table_tag.find_all("tr"):
+        cells = tr.find_all("td")
+        if not cells or len(cells) < 2:
+            continue
+
+        row_data = {}
+        for i, td in enumerate(cells):
+            key = headers[i] if i < len(headers) else f"column_{i}"
+
+            # Với td, nếu có span.link thì ưu tiên lấy text span
+            span_link = td.find("span", class_="link")
+            if span_link:
+                value = span_link.get_text(strip=True)
+            else:
+                # Giữ xuống dòng để không bị dính text
+                value = "\n".join([
+                    x.strip() for x in td.stripped_strings
+                ])
+            row_data[key] = value
+        rows.append(row_data)
+    return rows
+
+def extract_section_by_heading(soup, heading_text):
+    h2 = soup.find('h2', string=lambda t: t and heading_text in t)
+    if not h2:
+        return None
+
+    content = []
+    for sibling in h2.find_next_siblings():
+        if sibling.name == 'h2':
+            break
+        if sibling.name == 'table':
+            table_kv = extract_table_to_kv(sibling)
+            if table_kv:
+                content.append(table_kv)
+        elif sibling.name:
+            text = sibling.get_text(" ", strip=True)
+            if text:
+                content.append(text)
+    return content if content else None
+
+
+def call_dichvucong_api(service: str, ma_thu_tuc: str, label: str = None):
+    """
+    Gửi request tới https://dichvucong.gov.vn/jsp/rest.jsp với tham số tuỳ biến.
+
+    Args:
+        service (str): Tên dịch vụ, ví dụ: 'procedure_get_requires_by_procedure_id_service_v2'
+        ma_thu_tuc (str): ID thủ tục hành chính (VD: '5974')
+        cookies_dict (dict): Dict chứa cookies, gồm 'route', 'JSESSIONID', 'TS0115bee1'
+
+    Returns:
+        dict | str: Dữ liệu JSON nếu thành công, chuỗi lỗi nếu thất bại
+    """
+    url = "https://dichvucong.gov.vn/jsp/rest.jsp"
+
+    headers = {
+        "Accept": "application/json, text/javascript, */*; q=0.01",
+        "Accept-Language": "en,vi;q=0.9,ja;q=0.8,de;q=0.7,en-US;q=0.6,fr;q=0.5",
+        "Connection": "keep-alive",
+        "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+        "Origin": "https://dichvucong.gov.vn",
+        "Referer": f"https://dichvucong.gov.vn/p/home/dvc-tthc-thu-tuc-hanh-chinh-chi-tiet.html?ma_thu_tuc={ma_thu_tuc}",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-origin",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
+        "X-Requested-With": "XMLHttpRequest",
+        "sec-ch-ua": '"Google Chrome";v="137", "Chromium";v="137", "Not/A)Brand";v="24"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Windows"',
+    }
+
+    payload = {
+        "params": json.dumps({
+            "service": service,
+            "provider": "dvcquocgia",
+            "type": "ref",
+            "id": ma_thu_tuc,
+            "parent_id": ""
+        })
+    }
+
+    try:
+        response = requests.post(url, headers=headers, data=payload)
+        response.raise_for_status()
+        data = response.json()
+        
+        # Nếu cần gắn nhãn vào phần 'result'
+        if label and isinstance(data, dict) and 'result' in data and isinstance(data['result'], str):
+            data['result'] = f"{label}: {data['result']}"
+        
+        return data
+    except Exception as e:
+        return {"error": str(e)}
+
+
+import numpy as np
+
+def get_best_doc_from_sections(query_text, model_embed, sections_dict):
+    # Encode query
+    query_vec = model_embed.encode([query_text])
+    query_vec = query_vec / np.linalg.norm(query_vec, axis=1, keepdims=True)
+
+    texts = []
+    keys = []
+
+    for key, value in sections_dict.items():
+        if isinstance(value, list):
+            for v in value:
+                if isinstance(v, dict):
+                    text = f"{key}: " + " ".join(str(x) for x in v.values())
+                    texts.append(text)
+                    keys.append((key, text))
+                elif isinstance(v, str):
+                    text = f"{key}: {v}"
+                    texts.append(text)
+                    keys.append((key, text))
+        elif isinstance(value, str):
+            text = f"{key}: {value}"
+            texts.append(text)
+            keys.append((key, text))
+
+    if not texts:
+        return {
+            "best_section": None,
+            "best_doc": "",
+            "similarity": 0.0
+        }
+
+    # Encode văn bản
+    doc_vecs = model_embed.encode(texts)
+    doc_vecs = doc_vecs / np.linalg.norm(doc_vecs, axis=1, keepdims=True)
+
+    # Tính cosine similarity
+    similarities = np.dot(doc_vecs, query_vec.T).flatten()
+    similarities = np.atleast_1d(similarities)
+
+    # Lấy doc tốt nhất
+    best_idx = int(np.argmax(similarities))
+    best_key, best_text = keys[best_idx]
+
+    return {
+        "best_section": best_key,
+        "best_doc": best_text,
+        "similarity": float(similarities[best_idx])
+    }
+
+import requests
+
+def get_thu_tuc_html(ma_thu_tuc: str):
+    """
+    Tải trang HTML thủ tục hành chính từ dichvucong.gov.vn
+
+    Args:
+        ma_thu_tuc (str): Mã thủ tục (VD: "5974")
+        cookies_dict (dict): Cookie dạng dict gồm 'route', 'JSESSIONID', 'TS0115bee1'
+
+    Returns:
+        str: Nội dung HTML nếu thành công, hoặc chuỗi lỗi
+    """
+    url = f"https://dichvucong.gov.vn/p/home/dvc-tthc-thu-tuc-hanh-chinh-chi-tiet.html?ma_thu_tuc={ma_thu_tuc}"
+
+    headers = {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "Accept-Language": "en,vi;q=0.9,ja;q=0.8,de;q=0.7,en-US;q=0.6,fr;q=0.5",
+        "Cache-Control": "max-age=0",
+        "Connection": "keep-alive",
+        "Referer": "https://www.google.com/",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "cross-site",
+        "Sec-Fetch-User": "?1",
+        "Upgrade-Insecure-Requests": "1",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
+        "sec-ch-ua": '"Google Chrome";v="137", "Chromium";v="137", "Not/A)Brand";v="24"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Windows"',
+    }
+
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+        return response.text
+    except Exception as e:
+        return f"Lỗi khi tải HTML: {e}"
+
+
+def extract_thu_tuc(doc_text):
+    match = re.search(r"\*\*Tên thủ tục\*\*:\s*(.+)", doc_text)
+    if match:
+        return match.group(1).strip()
+    return None
+    
+def get_preferred_link(results):
+    """
+    Lấy link đầu tiên theo mẫu dichvucong.gov.vn, nếu không có thì lấy top-1 bất kỳ.
+    """
+    preferred_prefix = "https://dichvucong.gov.vn/p/home/dvc-tthc-thu-tuc-hanh-chinh-chi-tiet.html"
+    for r in results[:5]:
+        link = r.get('link', '')
+        if link.startswith(preferred_prefix):
+            return link
+    return results[0]['link'] if results else None
+
+def search_keyword(question):
+    api_key = "4d71a752a00145c83690f47006bc6a32c7bf717f08c0d0a997a064c7b381f226"
+    params = {
+        "engine": "google",
+        "q": f"site:dichvucong.gov.vn {question}",
+        "api_key": api_key,
+        "location": "Ho Chi Minh, Vietnam",
+        "hl": "vi",
+        "gl": "vn",
+        "num": 5   # Lấy 5 kết quả để lọc
+    }
+
+    search = GoogleSearch(params)
+    results = search.get_dict()
+    organic = results.get("organic_results", [])
+
+    if not organic:
+        return None, None, None
+
+    link = get_preferred_link(organic)
+    result = next((r for r in organic if r.get("link") == link), organic[0])
+    title = result.get("title", "")
+    snippet = result.get("snippet", "")
+
+    # Gộp để debug nếu cần
+    print(question + " " + title + " " + snippet, link)
+
+    return question + " " + title + " " + snippet, link, title, snippet
+    
+
+
+def respond(message, system_message, model_name):
+    message = message.lower().replace("online", "trực tuyến").replace("offline", "dịch vụ bưu chính")
+    if model_name == "Gemini":
+        print("Gemini")
+        query_text, link, title, snippet = search_keyword(message)
+        preferred_prefix = "https://dichvucong.gov.vn/p/home/dvc-tthc-thu-tuc-hanh-chinh-chi-tiet.html"
+        
+        if preferred_prefix in link:
+    
+            ma_thu_tuc = link.split("ma_thu_tuc=")[-1]
+            dieu_kien  = call_dichvucong_api("procedure_get_requires_by_procedure_id_service_v2", ma_thu_tuc, "Yêu cầu, điều kiện thực hiện: ")
+            trinh_tuc_thuc_hien = call_dichvucong_api("procedure_get_impl_orders_by_proc_id_service_v2", ma_thu_tuc, "Trình tự thực hiện: ")
+            html = get_thu_tuc_html(ma_thu_tuc)
+            soup = BeautifulSoup(html, 'html.parser')
+    
+            def flatten(lst):
+                return [item for sublist in lst for item in (sublist if isinstance(sublist, list) else [sublist])]
+            
+            sections = {
+                "dieu_kien": dieu_kien,
+                "trinh_tu_thuc_hien": trinh_tuc_thuc_hien,
+                "cach_thuc_thuc_hien": extract_section_by_heading(soup, "Cách thức thực hiện"),
+                "thanh_phan_ho_so":extract_section_by_heading(soup, "Thành phần hồ sơ"),
+                "can_cu_phap_ly":extract_section_by_heading(soup, "Căn cứ pháp lý"),
+                "ket_qua_thuc_hien": extract_section_by_heading(soup, "Kết quả thực hiện")
+            }
+    
+            best_doc_raw = get_best_doc_from_sections(query_text, model_embed, sections)['best_doc']
+    
+            if isinstance(best_doc_raw, dict):
+                best_doc_text = " ".join(f"{k}: {v}" for k, v in best_doc_raw.items())
+            else:
+                best_doc_text = str(best_doc_raw)
+            
+            best_doc_text += " " + snippet
+            print(best_doc_text)
+            
+    
+        else:
+            query_vec = model_embed.encode([query_text])
+            query_vec = query_vec / np.linalg.norm(query_vec, axis=1, keepdims=True)
+            
+            D, I = index.search(query_vec.astype(np.float32), k=10)
+        
+            retrieved_documents = []
+            pairs_for_rerank = []
+            
+            for rank, idx in enumerate(I[0]):
+                doc_text = metadata[idx]["text"]
+                print(idx)
+                print(doc_text[:100])
+                title_doc = extract_thu_tuc(doc_text)
+                if title_doc.lower().strip() ==  title.lower().strip():
+                    # Gắn idx vào metadata hoặc lưu riêng
+                    retrieved_documents= []
+                    pairs_for_rerank = []
+                    retrieved_documents.append((idx, doc_text, metadata[idx])) 
+                    pairs_for_rerank.append([query_text, doc_text]) 
+                    break
+                    
+            
+                # Gắn idx vào metadata hoặc lưu riêng
+                retrieved_documents.append((idx, doc_text, metadata[idx])) 
+                pairs_for_rerank.append([query_text, doc_text])       
+            
+            # ==== Bước 6: Rerank với CrossEncoder ====
+            
+            scores = rerank_model.predict(pairs_for_rerank)
+            scored_docs = list(zip(scores, retrieved_documents))
+            
+            # Sắp xếp theo score giảm dần
+            scored_docs.sort(reverse=True, key=lambda x: x[0])
+            
+            # Lấy document có score cao nhất
+            best_score, (best_idx, best_doc_text, best_meta) = scored_docs[0]
+            print("After rerank: ", best_doc_text[:100])
+        
+        
+        doc = best_doc_text
+        answer = generate_mcq(message, doc)
+        print(answer)
+        return f"""{answer}
+    
+            **Nguồn tham khảo:**  
+            1. {link}  
+            
+            """
+    else:
+        print("My Model")
+        final_prompt = f"""Bạn là một trợ lý AI hỗ trợ tra cứu dịch vụ công.
+        ## Câu hỏi: 
+        {message}"""
+        
+        messages2 = [{"role": "user", "content": final_prompt}]
+        text2 = tokenizer1.apply_chat_template(
+            messages2,
+            tokenize=False,
+            add_generation_prompt=True,
+            enable_thinking=False,
+        )
+        inputs2 = tokenizer1(text2, return_tensors="pt").to("cuda")
+        outputs2 = model1.generate(
+            **inputs2,
+            max_new_tokens=2048,
+            temperature=0.7,
+            top_p=0.8,
+            do_sample=True,
+        )
+        response = generate(tokenizer1.decode(outputs2[0], skip_special_tokens=True).split("assistant")[-1].replace("<think>", "").replace("</think>", "").strip())
+
+        print(response)
+    
+        return f"""{response}"""
+
+
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        chatbot = gr.Chatbot()
+
+    with gr.Row():
+        user_input = gr.Textbox(label="Nhập câu hỏi")
+        model_choice = gr.Radio(
+            choices=["Gemini", "My model"],
+            value="Gemini",
+            label="Chọn mô hình trả lời"
+        )
+        send_button = gr.Button("Gửi")
+    
+    state = gr.State([])
+
+    def chat_wrapper(message, model_name, history):
+        system_msg = ""
+        response = respond(message, system_msg, model_name)
+        history.append([message, response])
+        return history, history  # Trả về cho chatbot và cập nhật state
+
+    send_button.click(
+        chat_wrapper,
+        inputs=[user_input, model_choice, state],
+        outputs=[chatbot, state]
+    )
+
+demo.launch(share=True)

faiss_index_bo_tp_cosine_full2.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98114e3e804397389140aba2ff7de88e26a82559374ba5ae7615ac13100b3838
+size 58070061

faiss_metadata_bo_tp_full2.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd7c3907ede6446635fd6893720ccda81787cc2b2c1c7b9978784baa87e420a8
+size 65140375

requirements.txt

@@ -0,0 +1,4 @@
+unsloth
+faiss-gpu-cu11
+gradio
+google-search-results