1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
import clip
import torch
from torchvision import transforms
from ldm.models.diffusion.ddim import DDIMSampler
class CLIPEvaluator(object):
def __init__(self, device, clip_model='ViT-B/32') -> None:
self.device = device
self.model, clip_preprocess = clip.load(clip_model, device=self.device)
self.clip_preprocess = clip_preprocess
self.preprocess = transforms.Compose([transforms.Normalize(mean=[-1.0, -1.0, -1.0], std=[2.0, 2.0, 2.0])] + # Un-normalize from [-1.0, 1.0] (generator output) to [0, 1].
clip_preprocess.transforms[:2] + # to match CLIP input scale assumptions
clip_preprocess.transforms[4:]) # + skip convert PIL to tensor
def tokenize(self, strings: list):
return clip.tokenize(strings).to(self.device)
@torch.no_grad()
def encode_text(self, tokens: list) -> torch.Tensor:
return self.model.encode_text(tokens)
@torch.no_grad()
def encode_images(self, images: torch.Tensor) -> torch.Tensor:
images = self.preprocess(images).to(self.device)
return self.model.encode_image(images)
def get_text_features(self, text: str, norm: bool = True) -> torch.Tensor:
tokens = clip.tokenize(text).to(self.device)
text_features = self.encode_text(tokens).detach()
if norm:
text_features /= text_features.norm(dim=-1, keepdim=True)
return text_features
def get_image_features(self, img: torch.Tensor, norm: bool = True) -> torch.Tensor:
image_features = self.encode_images(img)
if norm:
image_features /= image_features.clone().norm(dim=-1, keepdim=True)
return image_features
def img_to_img_similarity(self, src_images, generated_images):
src_img_features = self.get_image_features(src_images)
gen_img_features = self.get_image_features(generated_images)
return (src_img_features @ gen_img_features.T).mean()
def txt_to_img_similarity(self, text, generated_images):
text_features = self.get_text_features(text)
gen_img_features = self.get_image_features(generated_images)
return (text_features @ gen_img_features.T).mean()
class LDMCLIPEvaluator(CLIPEvaluator):
def __init__(self, device, clip_model='ViT-B/32') -> None:
super().__init__(device, clip_model)
def evaluate(self, ldm_model, src_images, target_text, n_samples=64, n_steps=50):
sampler = DDIMSampler(ldm_model)
samples_per_batch = 8
n_batches = n_samples // samples_per_batch
# generate samples
all_samples=list()
with torch.no_grad():
with ldm_model.ema_scope():
uc = ldm_model.get_learned_conditioning(samples_per_batch * [""])
for batch in range(n_batches):
c = ldm_model.get_learned_conditioning(samples_per_batch * [target_text])
shape = [4, 256//8, 256//8]
samples_ddim, _ = sampler.sample(S=n_steps,
conditioning=c,
batch_size=samples_per_batch,
shape=shape,
verbose=False,
unconditional_guidance_scale=5.0,
unconditional_conditioning=uc,
eta=0.0)
x_samples_ddim = ldm_model.decode_first_stage(samples_ddim)
x_samples_ddim = torch.clamp(x_samples_ddim, min=-1.0, max=1.0)
all_samples.append(x_samples_ddim)
all_samples = torch.cat(all_samples, axis=0)
sim_samples_to_img = self.img_to_img_similarity(src_images, all_samples)
sim_samples_to_text = self.txt_to_img_similarity(target_text.replace("*", ""), all_samples)
return sim_samples_to_img, sim_samples_to_text
class ImageDirEvaluator(CLIPEvaluator):
def __init__(self, device, clip_model='ViT-B/32') -> None:
super().__init__(device, clip_model)
def evaluate(self, gen_samples, src_images, target_text):
sim_samples_to_img = self.img_to_img_similarity(src_images, gen_samples)
sim_samples_to_text = self.txt_to_img_similarity(target_text.replace("*", ""), gen_samples)
return sim_samples_to_img, sim_samples_to_text
|
