app.py

import os
import stat
import uuid
import subprocess
import tempfile
from zipfile import ZipFile
import gradio as gr
import spaces
from googletrans import Translator
from TTS.api import TTS
from faster_whisper import WhisperModel
import soundfile as sf
import numpy as np
import cv2
from huggingface_hub import HfApi

HF_TOKEN = os.environ.get("HF_TOKEN")
os.environ["COQUI_TOS_AGREED"] = "1"
api = HfApi(token=HF_TOKEN)
repo_id = "artificialguybr/video-dubbing"

# Extract FFmpeg
ZipFile("ffmpeg.zip").extractall()
st = os.stat('ffmpeg')
os.chmod('ffmpeg', st.st_mode | stat.S_IEXEC)

# Whisper model initialization
model_size = "small"
model = WhisperModel(model_size, device="cpu", compute_type="int8")

def check_for_faces(video_path):
    face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
    cap = cv2.VideoCapture(video_path)
    
    while True:
        ret, frame = cap.read()
        if not ret:
            break
        
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, 1.1, 4)
        
        if len(faces) > 0:
            return True
    
    return False

@spaces.GPU
def process_video(radio, video, target_language, has_closeup_face):
    if target_language is None:
        return gr.Error("Please select a Target Language for Dubbing.")
    
    run_uuid = uuid.uuid4().hex[:6]
    output_filename = f"{run_uuid}_resized_video.mp4"
    
    # Use FFmpeg via subprocess
    subprocess.run(['ffmpeg', '-i', video, '-vf', 'scale=-2:720', output_filename])
    
    video_path = output_filename
    if not os.path.exists(video_path):
        return f"Error: {video_path} does not exist."
    
    # Check video duration
    video_info = subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', video_path], capture_output=True, text=True)
    video_duration = float(video_info.stdout)
    
    if video_duration > 60:
        os.remove(video_path)
        return gr.Error("Video duration exceeds 1 minute. Please upload a shorter video.")
    
    # Extract audio
    subprocess.run(['ffmpeg', '-i', video_path, '-acodec', 'pcm_s24le', '-ar', '48000', '-map', 'a', f"{run_uuid}_output_audio.wav"])
    
    # Audio processing
    subprocess.run(['ffmpeg', '-y', '-i', f"{run_uuid}_output_audio.wav", '-af', 'lowpass=3000,highpass=100', f"{run_uuid}_output_audio_final.wav"])
    
    print("Attempting to transcribe with Whisper...")
    try:
        segments, info = model.transcribe(f"{run_uuid}_output_audio_final.wav", beam_size=5)
        whisper_text = " ".join(segment.text for segment in segments)
        whisper_language = info.language
        print(f"Transcription successful: {whisper_text}")
    except RuntimeError as e:
        print(f"RuntimeError encountered: {str(e)}")
        if "CUDA failed with error device-side assert triggered" in str(e):
            gr.Warning("Error. Space needs to restart. Please retry in a minute")
            api.restart_space(repo_id=repo_id)
    
    language_mapping = {'English': 'en', 'Spanish': 'es', 'French': 'fr', 'German': 'de', 'Italian': 'it', 'Portuguese': 'pt', 'Polish': 'pl', 'Turkish': 'tr', 'Russian': 'ru', 'Dutch': 'nl', 'Czech': 'cs', 'Arabic': 'ar', 'Chinese (Simplified)': 'zh-cn'}
    target_language_code = language_mapping[target_language]
    translator = Translator()
    translated_text = translator.translate(whisper_text, src=whisper_language, dest=target_language_code).text
    print(translated_text)
    
    tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
    tts.tts_to_file(translated_text, speaker_wav=f"{run_uuid}_output_audio_final.wav", file_path=f"{run_uuid}_output_synth.wav", language=target_language_code)
    
    has_face = check_for_faces(video_path) if not has_closeup_face else True
    
    if has_closeup_face:
        try:
            subprocess.run(['python', 'Wav2Lip/inference.py', '--checkpoint_path', 'Wav2Lip/checkpoints/wav2lip_gan.pth', '--face', video_path, '--audio', f'{run_uuid}_output_synth.wav', '--pads', '0', '15', '0', '0', '--resize_factor', '1', '--nosmooth', '--outfile', f'{run_uuid}_output_video.mp4'], check=True)
        except subprocess.CalledProcessError as e:
            if "Face not detected! Ensure the video contains a face in all the frames." in str(e.stderr):
                gr.Warning("Wav2lip didn't detect a face. Please try again with the option disabled.")
                subprocess.run(['ffmpeg', '-i', video_path, '-i', f'{run_uuid}_output_synth.wav', '-c:v', 'copy', '-c:a', 'aac', '-strict', 'experimental', '-map', '0:v:0', '-map', '1:a:0', f'{run_uuid}_output_video.mp4'])
            else:
                subprocess.run(['ffmpeg', '-i', video_path, '-i', f'{run_uuid}_output_synth.wav', '-c:v', 'copy', '-c:a', 'aac', '-strict', 'experimental', '-map', '0:v:0', '-map', '1:a:0', f'{run_uuid}_output_video.mp4'])
    
    if not os.path.exists(f"{run_uuid}_output_video.mp4"):
        raise FileNotFoundError(f"Error: {run_uuid}_output_video.mp4 was not generated.")
    
    output_video_path = f"{run_uuid}_output_video.mp4"
    
    # Cleanup
    files_to_delete = [
        f"{run_uuid}_resized_video.mp4",
        f"{run_uuid}_output_audio.wav",
        f"{run_uuid}_output_audio_final.wav",
        f"{run_uuid}_output_synth.wav"
    ]
    for file in files_to_delete:
        try:
            os.remove(file)
        except FileNotFoundError:
            print(f"File {file} not found for deletion.")
    
    return output_video_path

def swap(radio):
    if radio == "Upload":
        return gr.update(source="upload")
    else:
        return gr.update(source="webcam")

video = gr.Video()
radio = gr.Radio(["Upload", "Record"], value="Upload", show_label=False)
iface = gr.Interface(
    fn=process_video,
    inputs=[
        radio,
        video,
        gr.Dropdown(choices=["English", "Spanish", "French", "German", "Italian", "Portuguese", "Polish", "Turkish", "Russian", "Dutch", "Czech", "Arabic", "Chinese (Simplified)"], label="Target Language for Dubbing", value="Spanish"),
        gr.Checkbox(
            label="Video has a close-up face. Use Wav2lip.",
            value=False,
            info="Say if video have close-up face. For Wav2lip. Will not work if checked wrongly.")
    ],
    outputs=gr.Video(),
    live=False,
    title="AI Video Dubbing",
    description="""This tool was developed by [@artificialguybr](https://twitter.com/artificialguybr) using entirely open-source tools. Special thanks to Hugging Face for the GPU support. Thanks [@yeswondwer](https://twitter.com/@yeswondwerr) for original code. Test the [Video Transcription and Translate](https://huggingface.co/spaces/artificialguybr/VIDEO-TRANSLATION-TRANSCRIPTION) space!""",
    allow_flagging=False
)

with gr.Blocks() as demo:
    iface.render()
    radio.change(swap, inputs=[radio], outputs=video)
    gr.Markdown("""
    **Note:**
    - Video limit is 1 minute. It will dubbling all people using just one voice.
    - Generation may take up to 5 minutes.
    - By using this demo you agree to the terms of the Coqui Public Model License at https://coqui.ai/cpml
    - The tool uses open-source models for all models. It's a alpha version.
    - Quality can be improved but would require more processing time per video. For scalability and hardware limitations, speed was chosen, not just quality.
    - If you need more than 1 minute, duplicate the Space and change the limit on app.py.
    - If you incorrectly mark the 'Video has a close-up face' checkbox, the dubbing may not work as expected.
    """)

demo.queue()
demo.launch()