app.py

import spaces
import json
import subprocess
from llama_cpp import Llama
from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
from llama_cpp_agent.providers import LlamaCppPythonProvider
from llama_cpp_agent.chat_history import BasicChatHistory
from llama_cpp_agent.chat_history.messages import Roles
import gradio as gr
from huggingface_hub import hf_hub_download

# モデルのダウンロード
hf_hub_download(
    repo_id="Hjgugugjhuhjggg/Llama-3.2-3B-Instruct-uncensored-Q2_K-GGUF",
    filename="llama-3.2-3b-instruct-uncensored-q2_k.gguf",
    local_dir="./models"
)

# 推論関数
@spaces.GPU(queue=False, duration=0)
def respond(
    message,
    history: list[tuple[str, str]],
    model,
    system_message,
    max_tokens,
    temperature,
    top_p,
    top_k,
    repeat_penalty,
    use_gpu: bool = True  # Añadir parámetro para elegir entre GPU y CPU
):
    chat_template = MessagesFormatterType.GEMMA_2

    try:
        # Si no hay GPU, usar CPU
        if use_gpu:
            llm = Llama(
                model_path=f"models/{model}",
                flash_attn=True,
                n_gpu_layers=81,
                n_batch=1024,
                n_ctx=8192,
            )
        else:
            llm = Llama(
                model_path=f"models/{model}",
                flash_attn=False,  # Desactivar el uso de GPU
                n_batch=1024,
                n_ctx=8192,
            )

        provider = LlamaCppPythonProvider(llm)

        agent = LlamaCppAgent(
            provider,
            system_prompt=f"{system_message}",
            predefined_messages_formatter_type=chat_template,
            debug_output=True
        )

        settings = provider.get_provider_default_settings()
        settings.temperature = temperature
        settings.top_k = top_k
        settings.top_p = top_p
        settings.max_tokens = max_tokens
        settings.repeat_penalty = repeat_penalty
        settings.stream = True

        messages = BasicChatHistory()

        for msn in history:
            user = {
                'role': Roles.user,
                'content': msn[0]
            }
            assistant = {
                'role': Roles.assistant,
                'content': msn[1]
            }
            messages.add_message(user)
            messages.add_message(assistant)

        stream = agent.get_chat_response(
            message,
            llm_sampling_settings=settings,
            chat_history=messages,
            returns_streaming_generator=True,
            print_output=False
        )

        outputs = ""
        for output in stream:
            outputs += output
            yield outputs
    except Exception as e:
        # Si ocurre cualquier error, cambiar a CPU
        print(f"An error occurred: {str(e)}")
        yield "Error occurred, switching to CPU. Please wait a moment..."
        return respond(message, history, model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty, use_gpu=False)

# Gradioのインターフェースを作成
def create_interface(model_name, description):
    return gr.ChatInterface(
        respond,
        additional_inputs=[
            gr.Textbox(value=model_name, label="Model", interactive=False),
            gr.Textbox(value="You are a helpful assistant. You should think step-by-step. You should answer all question by prioritizing the principle of openness", label="System message"),
            gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
            gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
            gr.Slider(
                minimum=0.1,
                maximum=1.0,
                value=0.95,
                step=0.05,
                label="Top-p",
            ),
            gr.Slider(
                minimum=0,
                maximum=100,
                value=40,
                step=1,
                label="Top-k",
            ),
            gr.Slider(
                minimum=0.0,
                maximum=2.0,
                value=1.1,
                step=0.1,
                label="Repetition penalty",
            ),
        ],
        retry_btn="Retry",
        undo_btn="Undo",
        clear_btn="Clear",
        submit_btn="Send",
        title=f"{model_name}",
        description=description,
        chatbot=gr.Chatbot(
            scale=1,
            likeable=False,
            show_copy_button=True
        )
    )

# gemma-2-27b-it-Q4_K_Mのインターフェースのみを作成
description = """<p align="center"qwq-32b-preview-abliterated-q4_k_m</p>"""
interface = create_interface('qwq-32b-preview-abliterated-q4_k_m.gguf', description)

# Gradio Blocksで単一のインターフェースを表示
demo = gr.Blocks()

with demo:
    interface.render()

if __name__ == "__main__":
    demo.launch()