{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "64361b9e-39fe-4777-9887-68712aa1098b", "metadata": {}, "outputs": [], "source": [ "#Model Commit" ] }, { "cell_type": "code", "execution_count": 3, "id": "9483d5c4-b8c5-48f8-b86d-bde850edc532", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: huggingface_hub in /opt/conda/lib/python3.10/site-packages (0.21.3)\n", "Collecting python-dotenv\n", " Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2023.12.2)\n", "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.65.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (4.9.0)\n", "Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface_hub) (23.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2.0.4)\n", "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (1.26.18)\n", "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface_hub) (2023.11.17)\n", "Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", "Installing collected packages: python-dotenv\n", "Successfully installed python-dotenv-1.0.1\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0m" ] } ], "source": [ "!pip install huggingface_hub python-dotenv" ] }, { "cell_type": "code", "execution_count": 4, "id": "93c881e9-1081-4ab8-883e-c0bd8abb241e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Token has not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n", "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ], "source": [ "from dotenv import load_dotenv\n", "import os\n", "from huggingface_hub import login, hf_hub_download\n", "\n", "# Load environment variables from .env file\n", "load_dotenv()\n", "\n", "# Retrieve the Hugging Face token from the environment variable\n", "hf_token = os.getenv(\"HF_TOKEN\")\n", "\n", "# Ensure the token is present\n", "if not hf_token:\n", " raise ValueError(\"Hugging Face token not found. Make sure your .env file is configured correctly and HF_TOKEN is set.\")\n", "\n", "# Authenticate with Hugging Face using the token\n", "login(token=hf_token)" ] }, { "cell_type": "code", "execution_count": 7, "id": "6c7c4ca0-31fe-4dd5-ae42-136ddf947983", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting package metadata (current_repodata.json): done\n", "Solving environment: done\n", "\n", "\n", "==> WARNING: A newer version of conda exists. <==\n", " current version: 23.9.0\n", " latest version: 24.1.2\n", "\n", "Please update conda by running\n", "\n", " $ conda update -n base -c defaults conda\n", "\n", "Or to minimize the number of packages updated during conda update use\n", "\n", " conda install conda=24.1.2\n", "\n", "\n", "\n", "## Package Plan ##\n", "\n", " environment location: /opt/conda\n", "\n", " added / updated specs:\n", " - git\n", "\n", "\n", "The following packages will be downloaded:\n", "\n", " package | build\n", " ---------------------------|-----------------\n", " ca-certificates-2023.08.22 | h06a4308_0 130 KB anaconda\n", " certifi-2023.11.17 | py310h06a4308_0 159 KB anaconda\n", " curl-8.4.0 | hdbd6064_0 90 KB anaconda\n", " gdbm-1.18 | hd4cb3f1_4 193 KB anaconda\n", " gettext-0.21.0 | h39681ba_1 3.4 MB anaconda\n", " git-2.40.1 | pl5340h36fbf9e_1 9.9 MB\n", " libcurl-8.4.0 | h251f7ec_0 388 KB anaconda\n", " openssl-3.0.12 | h7f8727e_0 5.2 MB anaconda\n", " perl-5.34.0 | h5eee18b_2 13.3 MB anaconda\n", " ------------------------------------------------------------\n", " Total: 32.8 MB\n", "\n", "The following NEW packages will be INSTALLED:\n", "\n", " curl anaconda/linux-64::curl-8.4.0-hdbd6064_0 \n", " gdbm anaconda/linux-64::gdbm-1.18-hd4cb3f1_4 \n", " gettext anaconda/linux-64::gettext-0.21.0-h39681ba_1 \n", " git pkgs/main/linux-64::git-2.40.1-pl5340h36fbf9e_1 \n", " perl anaconda/linux-64::perl-5.34.0-h5eee18b_2 \n", "\n", "The following packages will be SUPERSEDED by a higher-priority channel:\n", "\n", " ca-certificates pkgs/main::ca-certificates-2023.12.12~ --> anaconda::ca-certificates-2023.08.22-h06a4308_0 \n", " certifi pkgs/main --> anaconda \n", " libcurl pkgs/main::libcurl-8.4.0-h251f7ec_1 --> anaconda::libcurl-8.4.0-h251f7ec_0 \n", " openssl pkgs/main --> anaconda \n", "\n", "\n", "\n", "Downloading and Extracting Packages:\n", "libcurl-8.4.0 | 388 KB | | 0% \n", "perl-5.34.0 | 13.3 MB | | 0% \u001b[A\n", "\n", "git-2.40.1 | 9.9 MB | | 0% \u001b[A\u001b[A\n", "\n", "\n", "certifi-2023.11.17 | 159 KB | | 0% \u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "openssl-3.0.12 | 5.2 MB | | 0% \u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "curl-8.4.0 | 90 KB | | 0% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "ca-certificates-2023 | 130 KB | | 0% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "gettext-0.21.0 | 3.4 MB | | 0% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "gdbm-1.18 | 193 KB | | 0% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "libcurl-8.4.0 | 388 KB | #5 | 4% \u001b[A\u001b[A\n", "\n", "\n", "certifi-2023.11.17 | 159 KB | ###7 | 10% \u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "openssl-3.0.12 | 5.2 MB | 1 | 0% \u001b[A\u001b[A\u001b[A\u001b[A\n", "perl-5.34.0 | 13.3 MB | | 0% \u001b[A\n", "\n", "git-2.40.1 | 9.9 MB | ##########6 | 29% \u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "ca-certificates-2023 | 130 KB | ####5 | 12% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "openssl-3.0.12 | 5.2 MB | #####7 | 16% \u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "curl-8.4.0 | 90 KB | ######6 | 18% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "perl-5.34.0 | 13.3 MB | #3 | 4% \u001b[A\n", "\n", "\n", "\n", "\n", "\n", "ca-certificates-2023 | 130 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "libcurl-8.4.0 | 388 KB | ##################################### | 100% \u001b[A\u001b[A\n", "\n", "\n", "certifi-2023.11.17 | 159 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\n", "\n", "\n", "certifi-2023.11.17 | 159 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "gettext-0.21.0 | 3.4 MB | 1 | 0% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "curl-8.4.0 | 90 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "openssl-3.0.12 | 5.2 MB | #######################7 | 64% \u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "gdbm-1.18 | 193 KB | ### | 8% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "perl-5.34.0 | 13.3 MB | ####6 | 12% \u001b[A\n", "perl-5.34.0 | 13.3 MB | ########6 | 23% \u001b[A\n", "perl-5.34.0 | 13.3 MB | ###############6 | 42% \u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "gdbm-1.18 | 193 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "gdbm-1.18 | 193 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "perl-5.34.0 | 13.3 MB | ########################3 | 66% \u001b[A\n", "perl-5.34.0 | 13.3 MB | ##############################8 | 83% \u001b[A\n", "\n", "git-2.40.1 | 9.9 MB | ##################################### | 100% \u001b[A\u001b[A\n", "\n", "git-2.40.1 | 9.9 MB | ##################################### | 100% \u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "gettext-0.21.0 | 3.4 MB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "gettext-0.21.0 | 3.4 MB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "openssl-3.0.12 | 5.2 MB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "openssl-3.0.12 | 5.2 MB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\u001b[A\n", "perl-5.34.0 | 13.3 MB | ##################################### | 100% \u001b[A\n", " \u001b[A\n", " \u001b[A\n", "\n", " \u001b[A\u001b[A\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n", "Preparing transaction: done\n", "Verifying transaction: done\n", "Executing transaction: done\n", "Collecting package metadata (current_repodata.json): / WARNING conda.models.version:get_matcher(556): Using .* with relational operator is superfluous and deprecated and will be removed in a future version of conda. Your spec was 1.7.1.*, but conda is ignoring the .* and treating it as 1.7.1\n", "done\n", "Solving environment: done\n", "\n", "\n", "==> WARNING: A newer version of conda exists. <==\n", " current version: 23.9.0\n", " latest version: 24.1.2\n", "\n", "Please update conda by running\n", "\n", " $ conda update -n base -c defaults conda\n", "\n", "Or to minimize the number of packages updated during conda update use\n", "\n", " conda install conda=24.1.2\n", "\n", "\n", "\n", "## Package Plan ##\n", "\n", " environment location: /opt/conda\n", "\n", " added / updated specs:\n", " - git-lfs\n", "\n", "\n", "The following packages will be downloaded:\n", "\n", " package | build\n", " ---------------------------|-----------------\n", " ca-certificates-2024.2.2 | hbcca054_0 152 KB conda-forge\n", " certifi-2024.2.2 | pyhd8ed1ab_0 157 KB conda-forge\n", " git-lfs-3.4.1 | ha770c72_0 3.6 MB conda-forge\n", " openssl-3.0.13 | h7f8727e_0 5.2 MB\n", " ------------------------------------------------------------\n", " Total: 9.1 MB\n", "\n", "The following NEW packages will be INSTALLED:\n", "\n", " git-lfs conda-forge/linux-64::git-lfs-3.4.1-ha770c72_0 \n", "\n", "The following packages will be UPDATED:\n", "\n", " ca-certificates anaconda::ca-certificates-2023.08.22-~ --> conda-forge::ca-certificates-2024.2.2-hbcca054_0 \n", " certifi anaconda/linux-64::certifi-2023.11.17~ --> conda-forge/noarch::certifi-2024.2.2-pyhd8ed1ab_0 \n", " openssl anaconda::openssl-3.0.12-h7f8727e_0 --> pkgs/main::openssl-3.0.13-h7f8727e_0 \n", "\n", "\n", "\n", "Downloading and Extracting Packages:\n", "certifi-2024.2.2 | 157 KB | | 0% \n", "git-lfs-3.4.1 | 3.6 MB | | 0% \u001b[A\n", "\n", "openssl-3.0.13 | 5.2 MB | | 0% \u001b[A\u001b[A\n", "\n", "\n", "ca-certificates-2024 | 152 KB | | 0% \u001b[A\u001b[A\u001b[A\n", "\n", "certifi-2024.2.2 | 157 KB | ###7 | 10% \u001b[A\u001b[A\n", "\n", "\n", "ca-certificates-2024 | 152 KB | ###9 | 11% \u001b[A\u001b[A\u001b[A\n", "git-lfs-3.4.1 | 3.6 MB | 1 | 0% \u001b[A\n", "\n", "\n", "certifi-2024.2.2 | 157 KB | ##################################### | 100% \u001b[A\u001b[A\u001b[A\n", "\n", "openssl-3.0.13 | 5.2 MB | ################ | 43% \u001b[A\u001b[A\n", "git-lfs-3.4.1 | 3.6 MB | ##############5 | 39% \u001b[A\n", "git-lfs-3.4.1 | 3.6 MB | ##################################### | 100% \u001b[A\n", "git-lfs-3.4.1 | 3.6 MB | ##################################### | 100% \u001b[A\n", "\n", "openssl-3.0.13 | 5.2 MB | ##################################### | 100% \u001b[A\u001b[A\n", "\n", " \u001b[A\u001b[A\n", " \u001b[A\n", "\n", " \u001b[A\u001b[A\n", "\n", "\n", " \u001b[A\u001b[A\u001b[A\n", "Preparing transaction: done\n", "Verifying transaction: done\n", "Executing transaction: done\n", "Git LFS initialized.\n", "Installation completed.\n" ] } ], "source": [ "import sys\n", "\n", "# Install git\n", "!conda install -c anaconda git -y\n", "\n", "# Install git-lfs\n", "!conda install -c conda-forge git-lfs -y\n", "\n", "# Set up git-lfs (you only need to do this once)\n", "!git lfs install\n", "\n", "print(\"Installation completed.\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "785e2b01-4046-4aac-b72b-3918ab723414", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:131: FutureWarning: 'Repository' (from 'huggingface_hub.repository') is deprecated and will be removed from version '1.0'. Please prefer the http-based alternatives instead. Given its large adoption in legacy code, the complete removal is only planned on next major release.\n", "For more details, please read https://huggingface.co/docs/huggingface_hub/concepts/git_vs_http.\n", " warnings.warn(warning_message, FutureWarning)\n" ] }, { "ename": "ValueError", "evalue": "If not specifying `clone_from`, you need to pass Repository a valid git clone.", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[8], line 33\u001b[0m\n\u001b[1;32m 31\u001b[0m repo \u001b[38;5;241m=\u001b[39m Repository(local_dir\u001b[38;5;241m=\u001b[39mrepo_local_path, clone_from\u001b[38;5;241m=\u001b[39mrepo_url, use_auth_token\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 32\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 33\u001b[0m repo \u001b[38;5;241m=\u001b[39m \u001b[43mRepository\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlocal_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_local_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muse_auth_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 34\u001b[0m repo\u001b[38;5;241m.\u001b[39mgit_pull()\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# Copy files from your model directory to the cloned repository\u001b[39;00m\n", "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:118\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[1;32m 116\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m smoothly_deprecate_use_auth_token(fn_name\u001b[38;5;241m=\u001b[39mfn\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, has_token\u001b[38;5;241m=\u001b[39mhas_token, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[0;32m--> 118\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:132\u001b[0m, in \u001b[0;36m_deprecate_method.._inner_deprecate_method..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 130\u001b[0m warning_message \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m message\n\u001b[1;32m 131\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(warning_message, \u001b[38;5;167;01mFutureWarning\u001b[39;00m)\n\u001b[0;32m--> 132\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/repository.py:538\u001b[0m, in \u001b[0;36mRepository.__init__\u001b[0;34m(self, local_dir, clone_from, repo_type, token, git_user, git_email, revision, skip_lfs_files, client)\u001b[0m\n\u001b[1;32m 536\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[Repository] is a valid git repo\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 537\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 538\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIf not specifying `clone_from`, you need to pass Repository a valid git clone.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 540\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhuggingface_token \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m (git_email \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m git_user \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 541\u001b[0m user \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient\u001b[38;5;241m.\u001b[39mwhoami(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhuggingface_token)\n", "\u001b[0;31mValueError\u001b[0m: If not specifying `clone_from`, you need to pass Repository a valid git clone." ] } ], "source": [ "from huggingface_hub import HfApi, HfFolder, Repository\n", "import os\n", "\n", "# Define the path to your model and the name of the Hugging Face repository\n", "local_model_directory = \"./workspace/outputs/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/a83a8c0d-5838-47ac-ba15-69da87e94619/epoch_3/step_final/\"\n", "repo_name = \"clau_creat\"\n", "username = \"arthrod\" # Replace with your Hugging Face username\n", "repo_id = f\"{username}/{repo_name}\"\n", "\n", "# Authenticate with Hugging Face\n", "#token = HfFolder.get_token()\n", "#if token is None:\n", "# raise ValueError(\"You must be logged in to Hugging Face. Use `huggingface-cli login`.\")\n", "\n", "# Initialize the HfApi object\n", "api = HfApi()\n", "\n", "# Check if the repository exists, if not, create it\n", "try:\n", " api.repo_info(repo_id=repo_id)\n", "except Exception as e:\n", " print(f\"Repository {repo_id} does not exist. Creating it...\")\n", " api.create_repo(token=token, name=repo_name, organization=username, private=False) # Set private=True for a private repository\n", "\n", "# Clone the repository (if necessary) and copy your model files into it\n", "repo_url = f\"https://huggingface.co/{repo_id}\"\n", "repo_local_path = os.path.join(os.getcwd(), repo_name)\n", "\n", "# If the repo directory doesn't exist, clone it. Otherwise, just pull the latest changes\n", "if not os.path.isdir(repo_local_path):\n", " repo = Repository(local_dir=repo_local_path, clone_from=repo_url, use_auth_token=True)\n", "else:\n", " repo = Repository(local_dir=repo_local_path, use_auth_token=True)\n", " repo.git_pull()\n", "\n", "# Copy files from your model directory to the cloned repository\n", "for file_name in os.listdir(local_model_directory):\n", " src_file_path = os.path.join(local_model_directory, file_name)\n", " dst_file_path = os.path.join(repo_local_path, file_name)\n", " if os.path.isfile(src_file_path):\n", " os.replace(src_file_path, dst_file_path)\n", "\n", "# Commit and push the changes to the repository\n", "repo.git_add(auto_lfs_track=True)\n", "repo.git_commit(\"Upload model to Hugging Face Hub\")\n", "repo.git_push()\n", "\n", "print(f\"Model successfully uploaded to: https://huggingface.co/{repo_id}\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "41eb533d-7d4b-45a6-ae1a-e6cab2275431", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting pickleshare\n", " Downloading pickleshare-0.7.5-py2.py3-none-any.whl.metadata (1.5 kB)\n", "Downloading pickleshare-0.7.5-py2.py3-none-any.whl (6.9 kB)\n", "Installing collected packages: pickleshare\n", "Successfully installed pickleshare-0.7.5\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0m" ] } ], "source": [ "!pip install pickleshare" ] }, { "cell_type": "code", "execution_count": 16, "id": "c6d17b7a-fb51-4e10-83b6-c47c4eabdf36", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[Errno 2] No such file or directory: '/outputs/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/a83a8c0d-5838-47ac-ba15-69da87e94619/epoch_3/step_final'\n", "/workspace/outputs/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/a83a8c0d-5838-47ac-ba15-69da87e94619/epoch_3\n" ] } ], "source": [ "%cd /outputs/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/a83a8c0d-5838-47ac-ba15-69da87e94619/epoch_3/step_final" ] }, { "cell_type": "code", "execution_count": 17, "id": "9e644c65-2937-42a6-b7b4-658f763b2b3a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/workspace/outputs/NousResearch/Nous-Hermes-2-Mistral-7B-DPO/a83a8c0d-5838-47ac-ba15-69da87e94619/epoch_3\n" ] } ], "source": [ "!pwd" ] }, { "cell_type": "code", "execution_count": null, "id": "be54ebe7-5970-46d2-b5dd-61da9670b1d9", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "83b5e3d4e3c34a7ea9f2ef61e59af30a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Upload 3 LFS files: 0%| | 0/3 [00:00