[Feature] Support AnimateDiff, a popular text2animation method

configs/animatediff/animatediff_Lyriel.py

@@ -0,0 +1,61 @@
+# config for model
+stable_diffusion_v15_url = 'runwayml/stable-diffusion-v1-5'
+models_path = '/home/AnimateDiff/models/'
+randomness = dict(
+    seed=[
+        10917152860782582783, 6399018107401806238, 15875751942533906793,
+        6653196880059936551
+    ],
+    diff_rank_seed=True)
+
+diffusion_scheduler = dict(
+    type='DDIMScheduler',
+    beta_end=0.012,
+    beta_schedule='linear',
+    beta_start=0.00085,
+    num_train_timesteps=1000,
+    prediction_type='epsilon',
+    set_alpha_to_one=True,
+    clip_sample=False,
+    thresholding=False,
+    steps_offset=1)
+
+model = dict(
+    type='AnimateDiff',
+    vae=dict(
+        type='AutoencoderKL',
+        from_pretrained=stable_diffusion_v15_url,
+        subfolder='vae'),
+    unet=dict(
+        type='UNet3DConditionMotionModel',
+        unet_use_cross_frame_attention=False,
+        unet_use_temporal_attention=False,
+        use_motion_module=True,
+        motion_module_resolutions=[1, 2, 4, 8],
+        motion_module_mid_block=False,
+        motion_module_decoder_only=False,
+        motion_module_type='Vanilla',
+        motion_module_kwargs=dict(
+            num_attention_heads=8,
+            num_transformer_block=1,
+            attention_block_types=['Temporal_Self', 'Temporal_Self'],
+            temporal_position_encoding=True,
+            temporal_position_encoding_max_len=24,
+            temporal_attention_dim_div=1),
+        subfolder='unet',
+        from_pretrained=stable_diffusion_v15_url),
+    text_encoder=dict(
+        type='ClipWrapper',
+        clip_type='huggingface',
+        pretrained_model_name_or_path=stable_diffusion_v15_url,
+        subfolder='text_encoder'),
+    tokenizer=stable_diffusion_v15_url,
+    scheduler=diffusion_scheduler,
+    test_scheduler=diffusion_scheduler,
+    data_preprocessor=dict(type='DataPreprocessor'),
+    motion_module_cfg=dict(path=models_path + 'Motion_Module/mm_sd_v14.ckpt'),
+    dream_booth_lora_cfg=dict(
+        type='ToonYou',
+        path=models_path + 'DreamBooth_LoRA/lyriel_v16.safetensors',
+        steps=25,
+        guidance_scale=7.5))

configs/animatediff/animatediff_MajicMix.py

@@ -0,0 +1,62 @@
+# config for model
+stable_diffusion_v15_url = 'runwayml/stable-diffusion-v1-5'
+models_path = '/home/AnimateDiff/models/'
+randomness = dict(
+    seed=[
+        1572448948722921032, 1099474677988590681, 6488833139725635347,
+        18339859844376517918
+    ],
+    diff_rank_seed=True)
+
+diffusion_scheduler = dict(
+    type='DDIMScheduler',
+    beta_end=0.012,
+    beta_schedule='linear',
+    beta_start=0.00085,
+    num_train_timesteps=1000,
+    prediction_type='epsilon',
+    set_alpha_to_one=True,
+    clip_sample=False,
+    thresholding=False,
+    steps_offset=1)
+
+model = dict(
+    type='AnimateDiff',
+    vae=dict(
+        type='AutoencoderKL',
+        from_pretrained=stable_diffusion_v15_url,
+        subfolder='vae'),
+    unet=dict(
+        type='UNet3DConditionMotionModel',
+        unet_use_cross_frame_attention=False,
+        unet_use_temporal_attention=False,
+        use_motion_module=True,
+        motion_module_resolutions=[1, 2, 4, 8],
+        motion_module_mid_block=False,
+        motion_module_decoder_only=False,
+        motion_module_type='Vanilla',
+        motion_module_kwargs=dict(
+            num_attention_heads=8,
+            num_transformer_block=1,
+            attention_block_types=['Temporal_Self', 'Temporal_Self'],
+            temporal_position_encoding=True,
+            temporal_position_encoding_max_len=24,
+            temporal_attention_dim_div=1),
+        subfolder='unet',
+        from_pretrained=stable_diffusion_v15_url),
+    text_encoder=dict(
+        type='ClipWrapper',
+        clip_type='huggingface',
+        pretrained_model_name_or_path=stable_diffusion_v15_url,
+        subfolder='text_encoder'),
+    tokenizer=stable_diffusion_v15_url,
+    scheduler=diffusion_scheduler,
+    test_scheduler=diffusion_scheduler,
+    data_preprocessor=dict(type='DataPreprocessor'),
+    motion_module_cfg=dict(path=models_path + 'Motion_Module/mm_sd_v14.ckpt'),
+    dream_booth_lora_cfg=dict(
+        type='ToonYou',
+        path=models_path +
+        'DreamBooth_LoRA/majicmixRealistic_v5Preview.safetensors',
+        steps=25,
+        guidance_scale=7.5))

configs/animatediff/animatediff_RcnzCartoon.py

@@ -0,0 +1,61 @@
+# config for model
+stable_diffusion_v15_url = 'runwayml/stable-diffusion-v1-5'
+models_path = '/home/AnimateDiff/models/'
+randomness = dict(
+    seed=[
+        16931037867122267877, 2094308009433392066, 4292543217695451092,
+        15572665120852309890
+    ],
+    diff_rank_seed=True)
+
+diffusion_scheduler = dict(
+    type='DDIMScheduler',
+    beta_end=0.012,
+    beta_schedule='linear',
+    beta_start=0.00085,
+    num_train_timesteps=1000,
+    prediction_type='epsilon',
+    set_alpha_to_one=True,
+    clip_sample=False,
+    thresholding=False,
+    steps_offset=1)
+
+model = dict(
+    type='AnimateDiff',
+    vae=dict(
+        type='AutoencoderKL',
+        from_pretrained=stable_diffusion_v15_url,
+        subfolder='vae'),
+    unet=dict(
+        type='UNet3DConditionMotionModel',
+        unet_use_cross_frame_attention=False,
+        unet_use_temporal_attention=False,
+        use_motion_module=True,
+        motion_module_resolutions=[1, 2, 4, 8],
+        motion_module_mid_block=False,
+        motion_module_decoder_only=False,
+        motion_module_type='Vanilla',
+        motion_module_kwargs=dict(
+            num_attention_heads=8,
+            num_transformer_block=1,
+            attention_block_types=['Temporal_Self', 'Temporal_Self'],
+            temporal_position_encoding=True,
+            temporal_position_encoding_max_len=24,
+            temporal_attention_dim_div=1),
+        subfolder='unet',
+        from_pretrained=stable_diffusion_v15_url),
+    text_encoder=dict(
+        type='ClipWrapper',
+        clip_type='huggingface',
+        pretrained_model_name_or_path=stable_diffusion_v15_url,
+        subfolder='text_encoder'),
+    tokenizer=stable_diffusion_v15_url,
+    scheduler=diffusion_scheduler,
+    test_scheduler=diffusion_scheduler,
+    data_preprocessor=dict(type='DataPreprocessor'),
+    motion_module_cfg=dict(path=models_path + 'Motion_Module/mm_sd_v14.ckpt'),
+    dream_booth_lora_cfg=dict(
+        type='ToonYou',
+        path=models_path + 'DreamBooth_LoRA/rcnzCartoon3d_v10.safetensors',
+        steps=25,
+        guidance_scale=7.5))

configs/animatediff/animatediff_RealisticVision.py

@@ -0,0 +1,62 @@
+# config for model
+stable_diffusion_v15_url = 'runwayml/stable-diffusion-v1-5'
+models_path = '/home/AnimateDiff/models/'
+randomness = dict(
+    seed=[
+        5658137986800322009, 12099779162349365895, 10499524853910852697,
+        16768009035333711932
+    ],
+    diff_rank_seed=True)
+
+diffusion_scheduler = dict(
+    type='DDIMScheduler',
+    beta_end=0.012,
+    beta_schedule='linear',
+    beta_start=0.00085,
+    num_train_timesteps=1000,
+    prediction_type='epsilon',
+    set_alpha_to_one=True,
+    clip_sample=False,
+    thresholding=False,
+    steps_offset=1)
+
+model = dict(
+    type='AnimateDiff',
+    vae=dict(
+        type='AutoencoderKL',
+        from_pretrained=stable_diffusion_v15_url,
+        subfolder='vae'),
+    unet=dict(
+        type='UNet3DConditionMotionModel',
+        unet_use_cross_frame_attention=False,
+        unet_use_temporal_attention=False,
+        use_motion_module=True,
+        motion_module_resolutions=[1, 2, 4, 8],
+        motion_module_mid_block=False,
+        motion_module_decoder_only=False,
+        motion_module_type='Vanilla',
+        motion_module_kwargs=dict(
+            num_attention_heads=8,
+            num_transformer_block=1,
+            attention_block_types=['Temporal_Self', 'Temporal_Self'],
+            temporal_position_encoding=True,
+            temporal_position_encoding_max_len=24,
+            temporal_attention_dim_div=1),
+        subfolder='unet',
+        from_pretrained=stable_diffusion_v15_url),
+    text_encoder=dict(
+        type='ClipWrapper',
+        clip_type='huggingface',
+        pretrained_model_name_or_path=stable_diffusion_v15_url,
+        subfolder='text_encoder'),
+    tokenizer=stable_diffusion_v15_url,
+    scheduler=diffusion_scheduler,
+    test_scheduler=diffusion_scheduler,
+    data_preprocessor=dict(type='DataPreprocessor'),
+    motion_module_cfg=dict(path=models_path + 'Motion_Module/mm_sd_v14.ckpt'),
+    dream_booth_lora_cfg=dict(
+        type='ToonYou',
+        path=models_path +
+        'DreamBooth_LoRA/realisticVisionV20_v20.safetensors',
+        steps=25,
+        guidance_scale=7.5))

configs/animatediff/animatediff_RealisticVision_v2.py

@@ -0,0 +1,64 @@
+# config for model
+stable_diffusion_v15_url = 'runwayml/stable-diffusion-v1-5'
+models_path = '/home/AnimateDiff/models/'
+randomness = dict(
+    seed=[
+        13100322578370451493, 14752961627088720670, 9329399085567825781,
+        16987697414827649302
+    ],
+    diff_rank_seed=True)
+
+diffusion_scheduler = dict(
+    type='DDIMScheduler',
+    beta_end=0.012,
+    beta_schedule='linear',
+    beta_start=0.00085,
+    num_train_timesteps=1000,
+    prediction_type='epsilon',
+    set_alpha_to_one=True,
+    clip_sample=False,
+    thresholding=False,
+    steps_offset=1)
+
+model = dict(
+    type='AnimateDiff',
+    vae=dict(
+        type='AutoencoderKL',
+        from_pretrained=stable_diffusion_v15_url,
+        subfolder='vae'),
+    unet=dict(
+        type='UNet3DConditionMotionModel',
+        use_inflated_groupnorm=True,
+        unet_use_cross_frame_attention=False,
+        unet_use_temporal_attention=False,
+        use_motion_module=True,
+        motion_module_resolutions=[1, 2, 4, 8],
+        motion_module_mid_block=True,
+        motion_module_decoder_only=False,
+        motion_module_type='Vanilla',
+        motion_module_kwargs=dict(
+            num_attention_heads=8,
+            num_transformer_block=1,
+            attention_block_types=['Temporal_Self', 'Temporal_Self'],
+            temporal_position_encoding=True,
+            temporal_position_encoding_max_len=32,
+            temporal_attention_dim_div=1),
+        subfolder='unet',
+        from_pretrained=stable_diffusion_v15_url),
+    text_encoder=dict(
+        type='ClipWrapper',
+        clip_type='huggingface',
+        pretrained_model_name_or_path=stable_diffusion_v15_url,
+        subfolder='text_encoder'),
+    tokenizer=stable_diffusion_v15_url,
+    scheduler=diffusion_scheduler,
+    test_scheduler=diffusion_scheduler,
+    data_preprocessor=dict(type='DataPreprocessor'),
+    motion_module_cfg=dict(path=models_path +
+                           'Motion_Module/mm_sd_v15_v2.ckpt'),
+    dream_booth_lora_cfg=dict(
+        type='ToonYou',
+        path=models_path +
+        'DreamBooth_LoRA/realisticVisionV20_v20.safetensors',
+        steps=25,
+        guidance_scale=7.5))

configs/animatediff/animatediff_ToonYou.py

@@ -0,0 +1,80 @@
+# config for model
+stable_diffusion_v15_url = 'runwayml/stable-diffusion-v1-5'
+models_path = '/home/AnimateDiff/models/'
+randomness = dict(
+    seed=[
+        10788741199826055526, 6520604954829636163, 6519455744612555650,
+        16372571278361863751
+    ],
+    diff_rank_seed=True)
+
+val_prompts = [
+    'best quality, masterpiece, 1girl, looking at viewer,\
+        blurry background, upper body, contemporary, dress',
+    'masterpiece, best quality, 1girl, solo, cherry blossoms,\
+        hanami, pink flower, white flower, spring season, wisteria,\
+            petals, flower, plum blossoms, outdoors, falling petals,\
+                white hair, black eyes,',
+    'best quality, masterpiece, 1boy, formal, abstract,\
+        looking at viewer, masculine, marble pattern',
+    'best quality, masterpiece, 1girl, cloudy sky,\
+        dandelion, contrapposto, alternate hairstyle,'
+]
+val_neg_propmts = [
+    '',
+    'badhandv4,easynegative,ng_deepnegative_v1_75t,verybadimagenegative_v1.3,\
+        bad-artist, bad_prompt_version2-neg, teeth',
+    '',
+    '',
+]
+diffusion_scheduler = dict(
+    type='DDIMScheduler',
+    beta_end=0.012,
+    beta_schedule='linear',
+    beta_start=0.00085,
+    num_train_timesteps=1000,
+    prediction_type='epsilon',
+    set_alpha_to_one=True,
+    clip_sample=False,
+    thresholding=False,
+    steps_offset=1)
+
+model = dict(
+    type='AnimateDiff',
+    vae=dict(
+        type='AutoencoderKL',
+        from_pretrained=stable_diffusion_v15_url,
+        subfolder='vae'),
+    unet=dict(
+        type='UNet3DConditionMotionModel',
+        unet_use_cross_frame_attention=False,
+        unet_use_temporal_attention=False,
+        use_motion_module=True,
+        motion_module_resolutions=[1, 2, 4, 8],
+        motion_module_mid_block=False,
+        motion_module_decoder_only=False,
+        motion_module_type='Vanilla',
+        motion_module_kwargs=dict(
+            num_attention_heads=8,
+            num_transformer_block=1,
+            attention_block_types=['Temporal_Self', 'Temporal_Self'],
+            temporal_position_encoding=True,
+            temporal_position_encoding_max_len=24,
+            temporal_attention_dim_div=1),
+        subfolder='unet',
+        from_pretrained=stable_diffusion_v15_url),
+    text_encoder=dict(
+        type='ClipWrapper',
+        clip_type='huggingface',
+        pretrained_model_name_or_path=stable_diffusion_v15_url,
+        subfolder='text_encoder'),
+    tokenizer=stable_diffusion_v15_url,
+    scheduler=diffusion_scheduler,
+    test_scheduler=diffusion_scheduler,
+    data_preprocessor=dict(type='DataPreprocessor'),
+    motion_module_cfg=dict(path=models_path + 'Motion_Module/mm_sd_v14.ckpt'),
+    dream_booth_lora_cfg=dict(
+        type='ToonYou',
+        path=models_path + 'DreamBooth_LoRA/toonyou_beta3.safetensors',
+        steps=25,
+        guidance_scale=7.5))