Upload 9 files

AutoModel.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3045413c560025a975f3a5d5ec93adb33adaaabf67603379a8e0c096d94b998
+size 3237240570

config.json

@@ -0,0 +1,28 @@
+{
+    "model_name": "AutoModel",
+    "hidden_size": 768,
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "intermediate_size": 3072,
+    "hidden_dropout_prob": 0.1,
+    "attention_probs_dropout_prob": 0.1,
+    "image_size": 224,
+    "image_channels": 3,
+    "patch_size": 16,
+    "max_position_embeddings": 512,
+    "vocab_size": 30522,
+    "type_vocab_size": 2,
+    "audio_sample_rate": 16000,
+    "audio_frame_size": 1024,
+    "audio_hop_size": 512,
+    "enable_vqa": true,
+    "enable_caption": true,
+    "enable_retrieval": true,
+    "enable_asr": true,
+    "enable_realtime_asr": true,
+    "batch_size": 32,
+    "learning_rate": 0.0001,
+    "weight_decay": 0.01,
+    "warmup_steps": 10000,
+    "max_steps": 100000
+}

model.py

@@ -0,0 +1,212 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import os
+# 配置类定义
+class Config:
+    def __init__(self):
+        # 模型架构参数
+        self.hidden_size = 768
+        self.num_attention_heads = 12
+        self.num_hidden_layers = 12
+        self.intermediate_size = 3072
+        self.hidden_dropout_prob = 0.1
+        self.attention_probs_dropout_prob = 0.1
+
+        # 图像相关
+        self.image_size = 224
+        self.image_channels = 3
+        self.patch_size = 16
+
+        # 文本相关
+        self.max_position_embeddings = 512
+        self.vocab_size = 30522
+        self.type_vocab_size = 2
+
+        # 语音相关
+        self.audio_sample_rate = 16000
+        self.audio_frame_size = 1024
+        self.audio_hop_size = 512
+
+        # 任务相关
+        self.enable_vqa = True
+        self.enable_caption = True
+        self.enable_retrieval = True
+        self.enable_asr = True  # 语音识别
+        self.enable_realtime_asr = True  # 实时语音识别
+
+        # 训练相关
+        self.batch_size = 32
+        self.learning_rate = 1e-4
+        self.weight_decay = 0.01
+        self.warmup_steps = 10000
+        self.max_steps = 100000
+
+# 模型相关类定义
+class ImageEncoder(nn.Module):
+    def __init__(self, config):
+        super(ImageEncoder, self).__init__()
+        self.config = config
+        self.encoder_layer = nn.Sequential(
+            nn.Conv2d(3, 64, kernel_size=3),
+            nn.ReLU(),
+            nn.MaxPool2d(2, 2),
+            nn.Flatten(),
+            nn.Linear(64 * 111 * 111, config.hidden_size)
+        )
+
+    def forward(self, image):
+        image_features = self.encoder_layer(image)
+        return image_features
+
+class TextEncoder(nn.Module):
+    def __init__(self, config):
+        super(TextEncoder, self).__init__()
+        self.config = config
+        self.transformer_layer = nn.TransformerEncoderLayer(
+            d_model=config.hidden_size, 
+            nhead=config.num_attention_heads,
+            batch_first=True
+        )
+        self.transformer_encoder = nn.TransformerEncoder(
+            self.transformer_layer, 
+            num_layers=config.num_hidden_layers
+        )
+
+    def forward(self, text):
+        text_features = self.transformer_encoder(text).mean(dim=1)
+        return text_features
+
+class AudioEncoder(nn.Module):
+    def __init__(self, config):
+        super(AudioEncoder, self).__init__()
+        self.config = config
+        self.encoder_layer = nn.Sequential(
+            nn.Linear(config.audio_sample_rate, config.hidden_size),
+            nn.ReLU(),
+            nn.Linear(config.hidden_size, config.hidden_size)
+        )
+
+    def forward(self, audio):
+        audio_features = self.encoder_layer(audio)
+        return audio_features
+
+class FusionLayer(nn.Module):
+    def __init__(self, config):
+        super(FusionLayer, self).__init__()
+        self.config = config
+        self.fusion_layer = nn.Linear(config.hidden_size * 3, config.hidden_size)
+
+    def forward(self, image_features, text_features, audio_features):
+        fused_features = torch.cat((image_features, text_features, audio_features), dim=1)
+        fused_features = self.fusion_layer(fused_features)
+        return fused_features
+
+class VQALayer(nn.Module):
+    def __init__(self, config):
+        super(VQALayer, self).__init__()
+        self.config = config
+        self.vqa_layer = nn.Linear(config.hidden_size, config.vocab_size)
+
+    def forward(self, fused_features):
+        vqa_output = self.vqa_layer(fused_features)
+        return vqa_output
+
+class CaptionLayer(nn.Module):
+    def __init__(self, config):
+        super(CaptionLayer, self).__init__()
+        self.config = config
+        self.caption_layer = nn.Linear(config.hidden_size, config.vocab_size)
+
+    def forward(self, fused_features):
+        caption_output = self.caption_layer(fused_features)
+        return caption_output
+
+class RetrievalLayer(nn.Module):
+    def __init__(self, config):
+        super(RetrievalLayer, self).__init__()
+        self.config = config
+        self.retrieval_layer = nn.Linear(config.hidden_size, config.vocab_size)
+
+    def forward(self, fused_features):
+        retrieval_output = self.retrieval_layer(fused_features)
+        return retrieval_output
+
+class ASRLayer(nn.Module):
+    def __init__(self, config):
+        super(ASRLayer, self).__init__()
+        self.config = config
+        self.asr_layer = nn.Linear(config.hidden_size, config.vocab_size)
+
+    def forward(self, fused_features):
+        asr_output = self.asr_layer(fused_features)
+        return asr_output
+
+class RealtimeASRLayer(nn.Module):
+    def __init__(self, config):
+        super(RealtimeASRLayer, self).__init__()
+        self.config = config
+        self.realtime_asr_layer = nn.Linear(config.hidden_size, config.vocab_size)
+
+    def forward(self, fused_features):
+        realtime_asr_output = self.realtime_asr_layer(fused_features)
+        return realtime_asr_output
+
+# 主模型定义
+class AutoModel(nn.Module):
+    def __init__(self, config):
+        super(AutoModel, self).__init__()
+        self.config = config
+        self.image_encoder = ImageEncoder(config)
+        self.text_encoder = TextEncoder(config)
+        self.audio_encoder = AudioEncoder(config)
+        self.fusion_layer = FusionLayer(config)
+        self.vqa_layer = VQALayer(config)
+        self.caption_layer = CaptionLayer(config)
+        self.retrieval_layer = RetrievalLayer(config)
+        self.asr_layer = ASRLayer(config)
+        self.realtime_asr_layer = RealtimeASRLayer(config)
+
+    def forward(self, image, text, audio):
+        image_features = self.image_encoder(image)
+        text_features = self.text_encoder(text)
+        audio_features = self.audio_encoder(audio)
+        fused_features = self.fusion_layer(image_features, text_features, audio_features)
+        vqa_output = self.vqa_layer(fused_features)
+        caption_output = self.caption_layer(fused_features)
+        retrieval_output = self.retrieval_layer(fused_features)
+        asr_output = self.asr_layer(fused_features)
+        realtime_asr_output = self.realtime_asr_layer(fused_features)
+        return vqa_output, caption_output, retrieval_output, asr_output, realtime_asr_output
+
+# 测试代码
+config = Config()
+model = AutoModel(config)
+image = torch.randn(1, 3, 224, 224)
+text = torch.randn(1, config.max_position_embeddings, config.hidden_size)
+audio = torch.randn(1, config.audio_sample_rate)
+vqa_output, caption_output, retrieval_output, asr_output, realtime_asr_output = model(image, text, audio)
+
+# 输出结果
+print("VQA output shape:", vqa_output.shape)
+print("Caption output shape:", caption_output.shape)
+print("Retrieval output shape:", retrieval_output.shape)
+print("ASR output shape:", asr_output.shape)
+print("Realtime ASR output shape:", realtime_asr_output.shape)
+
+# 打印总参数数量
+total_params = sum(p.numel() for p in model.parameters())
+print(f"\n总参数数量: {total_params}")
+
+# 定义保存路径
+save_dir = "./"  # 当前目录
+os.makedirs(save_dir, exist_ok=True)
+save_path = os.path.join(save_dir, "AutoModel.pth")
+
+# 保存模型权重
+torch.save(model.state_dict(), save_path)
+print(f"模型权重已保存到: {save_path}")
+
+
+
+

requirements.txt

@@ -0,0 +1,4 @@
+torch>=1.9.0
+transformers>=4.10.0
+numpy>=1.21.0
+gradio>=3.0.0

run_local.py

@@ -0,0 +1,63 @@
+import os
+import torch
+from model import AutoModel, Config
+
+def load_model(model_path, config_path):
+    """
+    加载模型权重和配置
+    """
+    # 加载配置
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"配置文件未找到: {config_path}")
+    print(f"加载配置文件: {config_path}")
+    config = Config()
+    
+    # 初始化模型
+    model = AutoModel(config)
+    
+    # 加载权重
+    if not os.path.exists(model_path):
+        raise FileNotFoundError(f"模型文件未找到: {model_path}")
+    print(f"加载模型权重: {model_path}")
+    state_dict = torch.load(model_path, map_location=torch.device("cpu"))
+    model.load_state_dict(state_dict)
+    model.eval()
+    print("模型加载成功并设置为评估模式。")
+    
+    return model, config
+
+
+def run_inference(model, config):
+    """
+    使用模型运行推理
+    """
+    # 模拟示例输入
+    image = torch.randn(1, 3, 224, 224)  # 图像输入
+    text = torch.randn(1, config.max_position_embeddings, config.hidden_size)  # 文本输入
+    audio = torch.randn(1, config.audio_sample_rate)  # 音频输入
+
+    # 模型推理
+    outputs = model(image, text, audio)
+    vqa_output, caption_output, retrieval_output, asr_output, realtime_asr_output = outputs
+
+    # 打印结果
+    print("\n推理结果:")
+    print(f"VQA output shape: {vqa_output.shape}")
+    print(f"Caption output shape: {caption_output.shape}")
+    print(f"Retrieval output shape: {retrieval_output.shape}")
+    print(f"ASR output shape: {asr_output.shape}")
+    print(f"Realtime ASR output shape: {realtime_asr_output.shape}")
+
+if __name__ == "__main__":
+    # 文件路径
+    model_path = "AutoModel.pth"
+    config_path = "config.json"
+
+    # 加载模型
+    try:
+        model, config = load_model(model_path, config_path)
+
+        # 运行推理
+        run_inference(model, config)
+    except Exception as e:
+        print(f"运行失败: {e}")

sky.py

@@ -0,0 +1,71 @@
+import json
+import os
+
+# 定义配置参数
+config_data = {
+    "hidden_size": 768,
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "intermediate_size": 3072,
+    "hidden_dropout_prob": 0.1,
+    "attention_probs_dropout_prob": 0.1,
+    "image_size": 224,
+    "image_channels": 3,
+    "patch_size": 16,
+    "max_position_embeddings": 512,
+    "vocab_size": 30522,
+    "type_vocab_size": 2,
+    "audio_sample_rate": 16000,
+    "audio_frame_size": 1024,
+    "audio_hop_size": 512,
+    "enable_vqa": True,
+    "enable_caption": True,
+    "enable_retrieval": True,
+    "enable_asr": True,
+    "enable_realtime_asr": True,
+    "batch_size": 32,
+    "learning_rate": 0.0001,
+    "weight_decay": 0.01,
+    "warmup_steps": 10000,
+    "max_steps": 100000
+}
+
+# 文件路径
+config_path = r"C:\Users\baby7\Desktop\zero_sg-pytorch-zero-v4\config.json"
+
+# 保存配置文件
+os.makedirs(os.path.dirname(config_path), exist_ok=True)
+with open(config_path, "w") as f:
+    json.dump(config_data, f, indent=4)
+
+print(f"配置文件已保存到: {config_path}")
+
+from transformers import BertTokenizer
+import os
+
+# 初始化分词器
+tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+
+# 保存分词器到目标路径
+tokenizer_path = r"C:\Users\baby7\Desktop\zero_sg-pytorch-zero-v4\tokenizer"
+os.makedirs(tokenizer_path, exist_ok=True)
+tokenizer.save_pretrained(tokenizer_path)
+
+print(f"分词器已保存到: {tokenizer_path}")
+
+
+#### **加载配置文件**
+from model import Config  # 假设您有Config类
+
+config_file = r"C:\Users\baby7\Desktop\zero_sg-pytorch-zero-v4\config.json"
+config = Config(config_file)
+print("加载的配置: ", config.__dict__)
+
+from transformers import BertTokenizer
+
+tokenizer_path = r"C:\Users\baby7\Desktop\zero_sg-pytorch-zero-v4\tokenizer"
+tokenizer = BertTokenizer.from_pretrained(tokenizer_path)
+text = "Hello, how are you?"
+encoded_input = tokenizer(text, return_tensors="pt", max_length=512, padding="max_length", truncation=True)
+
+print("分词器输出: ", encoded_input["input_ids"].shape)