Upload KeyVID model files
Browse files- upload_fast.py +98 -0
upload_fast.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
快速上传 KeyVID 模型到 Hugging Face Hub
|
| 4 |
+
针对大文件和高速上传优化
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from huggingface_hub import HfApi, upload_folder
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
import time
|
| 11 |
+
import os
|
| 12 |
+
|
| 13 |
+
# Model repository name on Hugging Face
|
| 14 |
+
MODEL_ID = "RyanWW/KeyVID"
|
| 15 |
+
KEYVID_PATH = "/dockerx/groups/KeyVID_hf_model"
|
| 16 |
+
|
| 17 |
+
def main():
|
| 18 |
+
print("🚀 KeyVID 快速上传到 Hugging Face")
|
| 19 |
+
print(f"📦 Repository: {MODEL_ID}")
|
| 20 |
+
print(f"📁 Directory: {KEYVID_PATH}\n")
|
| 21 |
+
|
| 22 |
+
# 检查认证
|
| 23 |
+
try:
|
| 24 |
+
api = HfApi()
|
| 25 |
+
print("✅ Hugging Face 认证成功\n")
|
| 26 |
+
except Exception as e:
|
| 27 |
+
print("❌ 需要 Hugging Face 认证")
|
| 28 |
+
print("请运行: huggingface-cli login")
|
| 29 |
+
print("或设置环境变量: export HF_TOKEN=your_token")
|
| 30 |
+
return
|
| 31 |
+
|
| 32 |
+
keyvid_dir = Path(KEYVID_PATH)
|
| 33 |
+
if not keyvid_dir.exists():
|
| 34 |
+
print(f"❌ 目录不存在: {KEYVID_PATH}")
|
| 35 |
+
return
|
| 36 |
+
|
| 37 |
+
# 优化的排除模式(只上传必要文件)
|
| 38 |
+
ignore_patterns = [
|
| 39 |
+
"**/__pycache__/**",
|
| 40 |
+
"**/.git/**",
|
| 41 |
+
"**/*.pyc",
|
| 42 |
+
"**/.DS_Store",
|
| 43 |
+
"**/save_results/**", # 排除结果文件
|
| 44 |
+
"**/*.log",
|
| 45 |
+
"**/*.tmp",
|
| 46 |
+
"**/upload*.py",
|
| 47 |
+
"**/.gitignore",
|
| 48 |
+
"**/.gitattributes",
|
| 49 |
+
]
|
| 50 |
+
|
| 51 |
+
# 显示上传策略
|
| 52 |
+
print("⚙️ 上传配置:")
|
| 53 |
+
print(" - multi_commits=True (分批提交,加快大文件)")
|
| 54 |
+
print(" - 自动处理大文件 (使用LFS)")
|
| 55 |
+
print(" - 跳过不必要的文件\n")
|
| 56 |
+
|
| 57 |
+
# 确认
|
| 58 |
+
response = input("❓ 开始上传? (y/n): ").strip().lower()
|
| 59 |
+
if response != 'y':
|
| 60 |
+
print("❌ 已取消")
|
| 61 |
+
return
|
| 62 |
+
|
| 63 |
+
print("\n⬆️ 开始上传...")
|
| 64 |
+
print("💡 提示: 大文件上传可能需要较长时间,请耐心等待\n")
|
| 65 |
+
|
| 66 |
+
start_time = time.time()
|
| 67 |
+
|
| 68 |
+
try:
|
| 69 |
+
upload_folder(
|
| 70 |
+
folder_path=str(keyvid_dir),
|
| 71 |
+
repo_id=MODEL_ID,
|
| 72 |
+
repo_type="model",
|
| 73 |
+
ignore_patterns=ignore_patterns,
|
| 74 |
+
commit_message="Upload KeyVID model files",
|
| 75 |
+
# 关键优化参数
|
| 76 |
+
multi_commits=True, # 分批提交,不需要等所有文件
|
| 77 |
+
multi_commits_verbose=True, # 显示详细进度
|
| 78 |
+
allow_patterns=None, # 不限制模式,上传所有匹配文件
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
elapsed = time.time() - start_time
|
| 82 |
+
print(f"\n✅ 上传完成!")
|
| 83 |
+
print(f"⏱️ 耗时: {elapsed/60:.2f} 分钟 ({elapsed:.2f} 秒)")
|
| 84 |
+
print(f"🔗 查看: https://huggingface.co/{MODEL_ID}")
|
| 85 |
+
|
| 86 |
+
except KeyboardInterrupt:
|
| 87 |
+
print("\n⚠️ 上传被用户中断")
|
| 88 |
+
print("💡 可以稍后重新运行脚本,Hugging Face 支持断点续传")
|
| 89 |
+
except Exception as e:
|
| 90 |
+
print(f"\n❌ 上传错误: {e}")
|
| 91 |
+
print("\n💡 建议:")
|
| 92 |
+
print(" 1. 检查网络连接")
|
| 93 |
+
print(" 2. 检查 Hugging Face token 是否有效")
|
| 94 |
+
print(" 3. 检查仓库权限 (repo_id: {MODEL_ID})")
|
| 95 |
+
print(" 4. 对于大文件,可能需要较长时间")
|
| 96 |
+
|
| 97 |
+
if __name__ == "__main__":
|
| 98 |
+
main()
|