update videocrafter2

app.py

@@ -21,27 +21,27 @@ def infer(prompt, output_size, seed, num_frames, ddim_steps, unconditional_guida
     window_size = 16
     window_stride = 4
     
-    # if output_size == "320x512":
-    #     width = 512 
-    #     height = 320
-    #     ckpt_dir_512 = "checkpoints/base_512_v1"
-    #     ckpt_path_512 = "checkpoints/base_512_v1/model_512.ckpt"
-    #     config_512 = "configs/inference_t2v_tconv512_v1.0_freenoise.yaml"
-    #     config_512 = OmegaConf.load(config_512)
-    #     model_config_512 = config_512.pop("model", OmegaConf.create())
-    #     model_512 = instantiate_from_config(model_config_512)
-    #     model_512 = model_512.cuda()
-    #     if not os.path.exists(ckpt_path_512):
-    #         os.makedirs(ckpt_dir_512, exist_ok=True)
-    #         hf_hub_download(repo_id="MoonQiu/LongerCrafter", filename="model_512.ckpt", local_dir=ckpt_dir_512)
-    #     try:
-    #         model_512 = load_model_checkpoint(model_512, ckpt_path_512)
-    #     except:
-    #         hf_hub_download(repo_id="MoonQiu/LongerCrafter", filename="model_512.ckpt", local_dir=ckpt_dir_512, force_download=True)
-    #         model_512 = load_model_checkpoint(model_512, ckpt_path_512)
-    #     model_512.eval()
-    #     model = model_512
-    #     fps = 8
+    if output_size == "320x512":
+        width = 512 
+        height = 320
+        ckpt_dir_512 = "checkpoints/base_512_v2"
+        ckpt_path_512 = "checkpoints/base_512_v2/model.ckpt"
+        config_512 = "configs/inference_t2v_tconv512_v2.0_freenoise.yaml"
+        config_512 = OmegaConf.load(config_512)
+        model_config_512 = config_512.pop("model", OmegaConf.create())
+        model_512 = instantiate_from_config(model_config_512)
+        model_512 = model_512.cuda()
+        if not os.path.exists(ckpt_path_512):
+            os.makedirs(ckpt_dir_512, exist_ok=True)
+            hf_hub_download(repo_id="VideoCrafter/VideoCrafter2", filename="model.ckpt", local_dir=ckpt_dir_512)
+        try:
+            model_512 = load_model_checkpoint(model_512, ckpt_path_512)
+        except:
+            hf_hub_download(repo_id="VideoCrafter/VideoCrafter2", filename="model.ckpt", local_dir=ckpt_dir_512, force_download=True)
+            model_512 = load_model_checkpoint(model_512, ckpt_path_512)
+        model_512.eval()
+        model = model_512
+        fps = 16
     if output_size == "576x1024":
         width = 1024
         height = 576
@@ -64,27 +64,27 @@ def infer(prompt, output_size, seed, num_frames, ddim_steps, unconditional_guida
         model = model_1024
         fps = 28
         num_frames = min(num_frames, 36)
-    # elif output_size == "256x256":
-    #     width = 256 
-    #     height = 256
-    #     ckpt_dir_256 = "checkpoints/base_256_v1"
-    #     ckpt_path_256 = "checkpoints/base_256_v1/model_256.pth"
-    #     config_256 = "configs/inference_t2v_tconv256_v1.0_freenoise.yaml"
-    #     config_256 = OmegaConf.load(config_256)
-    #     model_config_256 = config_256.pop("model", OmegaConf.create())
-    #     model_256 = instantiate_from_config(model_config_256)
-    #     model_256 = model_256.cuda()
-    #     if not os.path.exists(ckpt_path_256):
-    #         os.makedirs(ckpt_dir_256, exist_ok=True)
-    #         hf_hub_download(repo_id="MoonQiu/LongerCrafter", filename="model_256.pth", local_dir=ckpt_dir_256)
-    #     try:
-    #         model_256 = load_model_checkpoint(model_256, ckpt_path_256)
-    #     except:
-    #         hf_hub_download(repo_id="MoonQiu/LongerCrafter", filename="model_256.pth", local_dir=ckpt_dir_256, force_download=True)
-    #         model_256 = load_model_checkpoint(model_256, ckpt_path_256)
-    #     model_256.eval()
-    #     model = model_256
-    #     fps = 8
+    elif output_size == "256x256":
+        width = 256 
+        height = 256
+        ckpt_dir_256 = "checkpoints/base_256_v1"
+        ckpt_path_256 = "checkpoints/base_256_v1/model.ckpt"
+        config_256 = "configs/inference_t2v_tconv256_v1.0_freenoise.yaml"
+        config_256 = OmegaConf.load(config_256)
+        model_config_256 = config_256.pop("model", OmegaConf.create())
+        model_256 = instantiate_from_config(model_config_256)
+        model_256 = model_256.cuda()
+        if not os.path.exists(ckpt_path_256):
+            os.makedirs(ckpt_dir_256, exist_ok=True)
+            hf_hub_download(repo_id="VideoCrafter/Text2Video-256", filename="model.ckpt", local_dir=ckpt_dir_256)
+        try:
+            model_256 = load_model_checkpoint(model_256, ckpt_path_256)
+        except:
+            hf_hub_download(repo_id="VideoCrafter/Text2Video-256", filename="model.ckpt", local_dir=ckpt_dir_256, force_download=True)
+            model_256 = load_model_checkpoint(model_256, ckpt_path_256)
+        model_256.eval()
+        model = model_256
+        fps = 8
 
     if seed is None:
         seed = int.from_bytes(os.urandom(2), "big")
@@ -286,9 +286,7 @@ with gr.Blocks(css=css) as demo:
         with gr.Row():
             with gr.Accordion('FreeNoise Parameters (feel free to adjust these parameters based on your prompt): ', open=False):
                 with gr.Row():
-                    # output_size = gr.Dropdown(["320x512", "576x1024"], value="320x512", label="Output Size", info="250s for 512 model, 900s for 1024 model (32 frames). Recovering from sleeping will take more time to download ckpt")
-                    # output_size = gr.Dropdown(["256x256", "576x1024"], value="576x1024", label="Output Size", info="900s for 1024 model (32 frames). Recovering from sleeping will take more time to download ckpt")
-                    output_size = gr.Dropdown(["576x1024"], value="576x1024", label="Output Size", info="900s for 1024 model (32 frames). Recovering from sleeping will take more time to download ckpt")
+                    output_size = gr.Dropdown(["320x512", "576x1024", "256x256"], value="320x512", label="Output Size", info="250s for 512 model, 900s for 1024 model (32 frames). Recovering from sleeping will take more time to download ckpt")
                 with gr.Row():
                     num_frames = gr.Slider(label='Frames (a multiple of 4), max 36 for 1024 model',
                              minimum=16,

scripts/run_text2video_freenoise_256.sh

@@ -1,6 +1,6 @@
 name="base_256_test"
 
-ckpt='checkpoints/base_256_v1/model_256.pth'
+ckpt='checkpoints/base_256_v1/model.ckpt'
 config='configs/inference_t2v_tconv256_v1.0_freenoise.yaml'
 
 prompt_file="prompts/single_prompts.txt"

scripts/run_text2video_freenoise_512.sh

@@ -1,7 +1,7 @@
 name="base_512_test"
 
-ckpt='checkpoints/base_512_v1/model_512.ckpt'
-config='configs/inference_t2v_tconv512_v1.0_freenoise.yaml'
+ckpt='checkpoints/base_512_v2/model.ckpt'
+config='configs/inference_t2v_tconv512_v2.0_freenoise.yaml'
 
 prompt_file="prompts/single_prompts.txt"
 res_dir="results_freenoise_single_512"
@@ -18,7 +18,7 @@ python3 scripts/evaluation/inference_freenoise.py \
 --ddim_steps 50 \
 --ddim_eta 0.0 \
 --prompt_file $prompt_file \
---fps 8 \
+--fps 16 \
 --frames 64 \
 --window_size 16 \
 --window_stride 4 

scripts/run_text2video_freenoise_mp_256.sh

@@ -1,6 +1,6 @@
 name="base_256_test"
 
-ckpt='checkpoints/base_256_v1/model_256.pth'
+ckpt='checkpoints/base_256_v1/model.ckpt'
 config='configs/inference_t2v_tconv256_v1.0_freenoise.yaml'
 
 prompt_file="prompts/mp_prompts.txt"

scripts/run_text2video_freenoise_mp_512.sh

@@ -1,7 +1,7 @@
 name="base_512_test"
 
-ckpt='checkpoints/base_512_v1/model_512.ckpt'
-config='configs/inference_t2v_tconv512_v1.0_freenoise.yaml'
+ckpt='checkpoints/base_512_v2/model.ckpt'
+config='configs/inference_t2v_tconv512_v2.0_freenoise.yaml'
 
 prompt_file="prompts/mp_prompts.txt"
 res_dir="results_freenoise_mp_512"
@@ -18,7 +18,7 @@ python3 scripts/evaluation/inference_freenoise_mp.py \
 --ddim_steps 50 \
 --ddim_eta 0.0 \
 --prompt_file $prompt_file \
---fps 8 \
+--fps 16 \
 --frames 64 \
 --window_size 16 \
 --window_stride 4