import os import boto3 import gdown import tempfile import shutil from dotenv import load_dotenv from utils.utils import upload_file load_dotenv() # Environment variables R2_ACCESS_KEY = os.getenv('R2_ACCESS_KEY') R2_SECRET_KEY = os.getenv('R2_SECRET_KEY') R2_BUCKET_NAME = os.getenv('R2_BUCKET_NAME') R2_ENDPOINT_URL = os.getenv('R2_ENDPOINT_URL') def download_from_google_folder(url): # Create a temporary directory with tempfile.TemporaryDirectory() as download_dir: print(f'Downloading folder to temporary directory: {download_dir}') # Download the entire folder gdown.download_folder(url, output=download_dir, quiet=False) res = [] # Upload files to R2 for root, _, files in os.walk(download_dir): for file_name in files: file_path = os.path.join(root, file_name) object_name = os.path.relpath(file_path, download_dir) print(f'Uploading file: {file_path}, object name: {object_name}') upload_file(file_path, R2_BUCKET_NAME, object_name, R2_ENDPOINT_URL, R2_ACCESS_KEY, R2_SECRET_KEY) res.append(f'https://pub-08a118f4cb7c4b208b55e6877b0bacca.r2.dev/warden-ai/{object_name}') print(res) return res