Upload 2 files
Browse files
README.md
CHANGED
@@ -1,19 +1,19 @@
|
|
1 |
-
---
|
2 |
-
title: 'FABRIC: Personalizing Diffusion Models with Iterative Feedback'
|
3 |
-
emoji: π¨
|
4 |
-
colorFrom: blue
|
5 |
-
colorTo: purple
|
6 |
-
sdk: gradio
|
7 |
-
sdk_version:
|
8 |
-
app_file: app.py
|
9 |
-
pinned: false
|
10 |
-
license: apache-2.0
|
11 |
-
---
|
12 |
-
|
13 |
-
Demo for Arxiv paper at https://arxiv.org/abs/2307.10159
|
14 |
-
|
15 |
-
Official code: https://github.com/sd-fabric/fabric
|
16 |
-
|
17 |
-
Official AUTOMATIC1111 plugin: https://github.com/dvruette/sd-webui-fabric
|
18 |
-
|
19 |
Unofficial ComfyUI node: https://github.com/ssitu/ComfyUI_fabric
|
|
|
1 |
+
---
|
2 |
+
title: 'FABRIC: Personalizing Diffusion Models with Iterative Feedback'
|
3 |
+
emoji: π¨
|
4 |
+
colorFrom: blue
|
5 |
+
colorTo: purple
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 4.41.0
|
8 |
+
app_file: app.py
|
9 |
+
pinned: false
|
10 |
+
license: apache-2.0
|
11 |
+
---
|
12 |
+
|
13 |
+
Demo for Arxiv paper at https://arxiv.org/abs/2307.10159
|
14 |
+
|
15 |
+
Official code: https://github.com/sd-fabric/fabric
|
16 |
+
|
17 |
+
Official AUTOMATIC1111 plugin: https://github.com/dvruette/sd-webui-fabric
|
18 |
+
|
19 |
Unofficial ComfyUI node: https://github.com/ssitu/ComfyUI_fabric
|
app.py
CHANGED
@@ -1,278 +1,280 @@
|
|
1 |
-
import
|
2 |
-
import
|
3 |
-
import
|
4 |
-
|
5 |
-
|
6 |
-
import
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
model_name = ""
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
self.
|
22 |
-
self.
|
23 |
-
self.
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
color: #
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
color: #
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
color: #
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
color: #
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
if
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
"
|
134 |
-
"
|
135 |
-
"
|
136 |
-
"
|
137 |
-
"
|
138 |
-
"
|
139 |
-
"
|
140 |
-
"
|
141 |
-
"
|
142 |
-
"
|
143 |
-
"
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
|
161 |
-
|
162 |
-
if
|
163 |
-
all_imgs
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
|
169 |
-
all_imgs
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
|
207 |
-
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
|
212 |
-
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
-
|
219 |
-
|
220 |
-
|
221 |
-
|
222 |
-
|
223 |
-
|
224 |
-
|
225 |
-
|
226 |
-
|
227 |
-
|
228 |
-
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
|
247 |
-
|
248 |
-
|
249 |
-
|
250 |
-
|
251 |
-
|
252 |
-
|
253 |
-
|
254 |
-
|
255 |
-
|
256 |
-
|
257 |
-
|
258 |
-
|
259 |
-
|
260 |
-
|
261 |
-
|
262 |
-
|
263 |
-
|
264 |
-
|
265 |
-
for i,
|
266 |
-
|
267 |
-
|
268 |
-
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
273 |
-
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
|
|
|
|
278 |
demo.launch(debug=True)
|
|
|
1 |
+
import spaces
|
2 |
+
import functools
|
3 |
+
import random
|
4 |
+
import logging
|
5 |
+
|
6 |
+
import gradio as gr
|
7 |
+
import torch
|
8 |
+
|
9 |
+
from fabric.generator import AttentionBasedGenerator
|
10 |
+
|
11 |
+
logging.basicConfig(level=logging.INFO)
|
12 |
+
logger = logging.getLogger(__name__)
|
13 |
+
|
14 |
+
|
15 |
+
#model_name = "dreamlike-art/dreamlike-photoreal-2.0"
|
16 |
+
model_name = ""
|
17 |
+
model_ckpt = "https://huggingface.co/Lykon/DreamShaper/blob/main/DreamShaper_7_pruned.safetensors"
|
18 |
+
|
19 |
+
class GeneratorWrapper:
|
20 |
+
def __init__(self, model_name=None, model_ckpt=None):
|
21 |
+
self.model_name = model_name if model_name else None
|
22 |
+
self.model_ckpt = model_ckpt if model_ckpt else None
|
23 |
+
self.dtype = torch.float16 if torch.cuda.is_available() else torch.float32
|
24 |
+
self.device = "cuda" if torch.cuda.is_available() else "cpu"
|
25 |
+
|
26 |
+
self.reload()
|
27 |
+
|
28 |
+
def generate(self, *args, **kwargs):
|
29 |
+
if not hasattr(self, "generator"):
|
30 |
+
self.reload()
|
31 |
+
return self.generator.generate(*args, **kwargs)
|
32 |
+
|
33 |
+
def to(self, device):
|
34 |
+
return self.generator.to(device)
|
35 |
+
|
36 |
+
def reload(self):
|
37 |
+
if hasattr(self, "generator"):
|
38 |
+
del self.generator
|
39 |
+
if self.device == "cuda":
|
40 |
+
torch.cuda.empty_cache()
|
41 |
+
self.generator = AttentionBasedGenerator(
|
42 |
+
model_name=self.model_name,
|
43 |
+
model_ckpt=self.model_ckpt,
|
44 |
+
torch_dtype=self.dtype,
|
45 |
+
).to(self.device)
|
46 |
+
|
47 |
+
generator = GeneratorWrapper(model_name, model_ckpt)
|
48 |
+
|
49 |
+
|
50 |
+
css = """
|
51 |
+
.btn-green {
|
52 |
+
background-image: linear-gradient(to bottom right, #86efac, #22c55e) !important;
|
53 |
+
border-color: #22c55e !important;
|
54 |
+
color: #166534 !important;
|
55 |
+
}
|
56 |
+
.btn-green:hover {
|
57 |
+
background-image: linear-gradient(to bottom right, #86efac, #86efac) !important;
|
58 |
+
}
|
59 |
+
.btn-red {
|
60 |
+
background: linear-gradient(to bottom right, #fda4af, #fb7185) !important;
|
61 |
+
border-color: #fb7185 !important;
|
62 |
+
color: #9f1239 !important;
|
63 |
+
}
|
64 |
+
.btn-red:hover {background: linear-gradient(to bottom right, #fda4af, #fda4af) !important;}
|
65 |
+
|
66 |
+
/*****/
|
67 |
+
|
68 |
+
.dark .btn-green {
|
69 |
+
background-image: linear-gradient(to bottom right, #047857, #065f46) !important;
|
70 |
+
border-color: #047857 !important;
|
71 |
+
color: #ffffff !important;
|
72 |
+
}
|
73 |
+
.dark .btn-green:hover {
|
74 |
+
background-image: linear-gradient(to bottom right, #047857, #047857) !important;
|
75 |
+
}
|
76 |
+
.dark .btn-red {
|
77 |
+
background: linear-gradient(to bottom right, #be123c, #9f1239) !important;
|
78 |
+
border-color: #be123c !important;
|
79 |
+
color: #ffffff !important;
|
80 |
+
}
|
81 |
+
.dark .btn-red:hover {background: linear-gradient(to bottom right, #be123c, #be123c) !important;}
|
82 |
+
"""
|
83 |
+
|
84 |
+
@spaces.GPU(duration=59)
|
85 |
+
def generate_fn(
|
86 |
+
feedback_enabled,
|
87 |
+
max_feedback_imgs,
|
88 |
+
prompt,
|
89 |
+
neg_prompt,
|
90 |
+
liked,
|
91 |
+
disliked,
|
92 |
+
denoising_steps,
|
93 |
+
guidance_scale,
|
94 |
+
feedback_start,
|
95 |
+
feedback_end,
|
96 |
+
min_weight,
|
97 |
+
max_weight,
|
98 |
+
neg_scale,
|
99 |
+
batch_size,
|
100 |
+
seed,
|
101 |
+
progress=gr.Progress(track_tqdm=True),
|
102 |
+
):
|
103 |
+
try:
|
104 |
+
if seed < 0:
|
105 |
+
seed = random.randint(1,9999999999999999) #16 digits is an arbitrary limit
|
106 |
+
|
107 |
+
max_feedback_imgs = max(0, int(max_feedback_imgs))
|
108 |
+
total_images = (len(liked) if liked else 0) + (len(disliked) if disliked else 0)
|
109 |
+
|
110 |
+
if not feedback_enabled:
|
111 |
+
liked = []
|
112 |
+
disliked = []
|
113 |
+
elif total_images > max_feedback_imgs:
|
114 |
+
if liked and disliked:
|
115 |
+
max_disliked = min(len(disliked), max_feedback_imgs // 2)
|
116 |
+
max_liked = min(len(liked), max_feedback_imgs - max_disliked)
|
117 |
+
if max_liked > len(liked):
|
118 |
+
max_disliked = max_feedback_imgs - max_liked
|
119 |
+
liked = liked[-max_liked:]
|
120 |
+
disliked = disliked[-max_disliked:]
|
121 |
+
elif liked:
|
122 |
+
liked = liked[-max_feedback_imgs:]
|
123 |
+
disliked = []
|
124 |
+
else:
|
125 |
+
liked = []
|
126 |
+
disliked = disliked[-max_feedback_imgs:]
|
127 |
+
# else: keep all feedback images
|
128 |
+
|
129 |
+
logger.info(f"Generate images: {prompt=}, {neg_prompt=}, {len(liked)=}, {len(disliked)=}, {max_feedback_imgs=}, {batch_size=}, {seed=}")
|
130 |
+
logger.info(f"Params: {denoising_steps=}, {guidance_scale=}, {feedback_start=}, {feedback_end=}, {min_weight=}, {max_weight=}, {neg_scale=}")
|
131 |
+
|
132 |
+
generate_kwargs = {
|
133 |
+
"prompt": prompt,
|
134 |
+
"negative_prompt": neg_prompt,
|
135 |
+
"liked": liked,
|
136 |
+
"disliked": disliked,
|
137 |
+
"denoising_steps": denoising_steps,
|
138 |
+
"guidance_scale": guidance_scale,
|
139 |
+
"feedback_start": feedback_start,
|
140 |
+
"feedback_end": feedback_end,
|
141 |
+
"min_weight": min_weight,
|
142 |
+
"max_weight": max_weight,
|
143 |
+
"neg_scale": neg_scale,
|
144 |
+
"seed": seed,
|
145 |
+
"n_images": batch_size,
|
146 |
+
}
|
147 |
+
|
148 |
+
try:
|
149 |
+
images = generator.generate(**generate_kwargs)
|
150 |
+
except RuntimeError as err:
|
151 |
+
if 'out of memory' in str(err):
|
152 |
+
generator.reload()
|
153 |
+
logger.info(f"Ran out of memory trying to generate {batch_size=} images with {len(liked)=} and {len(disliked)=} feedback images.")
|
154 |
+
raise
|
155 |
+
return [(img, f"Image {i+1}") for i, img in enumerate(images)], images, seed
|
156 |
+
except Exception as err:
|
157 |
+
logger.error(err)
|
158 |
+
raise gr.Error(str(err))
|
159 |
+
|
160 |
+
|
161 |
+
def add_img_from_list(i, curr_imgs, all_imgs):
|
162 |
+
if all_imgs is None:
|
163 |
+
all_imgs = []
|
164 |
+
if i >= 0 and i < len(curr_imgs):
|
165 |
+
all_imgs.append(curr_imgs[i])
|
166 |
+
return all_imgs, all_imgs # return (gallery, state)
|
167 |
+
|
168 |
+
def add_img(img, all_imgs):
|
169 |
+
if all_imgs is None:
|
170 |
+
all_imgs = []
|
171 |
+
all_imgs.append(img)
|
172 |
+
return None, all_imgs, all_imgs
|
173 |
+
|
174 |
+
def remove_img_from_list(event: gr.SelectData, imgs):
|
175 |
+
if event.index >= 0 and event.index < len(imgs):
|
176 |
+
imgs.pop(event.index)
|
177 |
+
return imgs, imgs
|
178 |
+
|
179 |
+
def duplicate_seed_value(seed): #I don't like the progress bar showing on the previous seed box and this is how I hide it
|
180 |
+
return seed
|
181 |
+
|
182 |
+
with gr.Blocks(css=css) as demo:
|
183 |
+
|
184 |
+
liked_imgs = gr.State([])
|
185 |
+
disliked_imgs = gr.State([])
|
186 |
+
curr_imgs = gr.State([])
|
187 |
+
|
188 |
+
with gr.Row():
|
189 |
+
with gr.Column(scale=100):
|
190 |
+
prompt = gr.Textbox(label="Prompt")
|
191 |
+
neg_prompt = gr.Textbox(label="Negative prompt", value="lowres, bad anatomy, bad hands, cropped, worst quality")
|
192 |
+
submit_btn = gr.Button("Generate", variant="primary", min_width="96px")
|
193 |
+
|
194 |
+
with gr.Row(equal_height=False):
|
195 |
+
with gr.Column():
|
196 |
+
denoising_steps = gr.Slider(1, 100, value=20, step=1, label="Sampling steps")
|
197 |
+
guidance_scale = gr.Slider(0.0, 30.0, value=6, step=0.25, label="CFG scale")
|
198 |
+
batch_size = gr.Slider(1, 10, value=4, step=1, label="Batch size", interactive=False)
|
199 |
+
seed = gr.Number(-1, minimum=-1, precision=0, label="Seed")
|
200 |
+
max_feedback_imgs = gr.Slider(0, 20, value=6, step=1, label="Max. feedback images", info="Maximum number of liked/disliked images to be used. If exceeded, only the most recent images will be used as feedback. (NOTE: large number of feedback imgs => high VRAM requirements)")
|
201 |
+
feedback_enabled = gr.Checkbox(True, label="Enable feedback", interactive=True)
|
202 |
+
|
203 |
+
with gr.Accordion("Liked Images", open=True):
|
204 |
+
liked_img_input = gr.Image(type="pil", shape=(512, 512), height=128, label="Upload liked image")
|
205 |
+
like_gallery = gr.Gallery(label="π Liked images (click to remove)", columns=[3, 4, 3, 4, 5, 6], height=256, allow_preview=False)
|
206 |
+
clear_liked_btn = gr.Button("Clear likes")
|
207 |
+
|
208 |
+
with gr.Accordion("Disliked Images", open=True):
|
209 |
+
disliked_img_input = gr.Image(type="pil", shape=(512, 512), height=128, label="Upload disliked image")
|
210 |
+
dislike_gallery = gr.Gallery(label="π Disliked images (click to remove)", columns=[3, 4, 3, 4, 5, 6], height=256, allow_preview=False)
|
211 |
+
clear_disliked_btn = gr.Button("Clear dislikes")
|
212 |
+
|
213 |
+
with gr.Accordion("Feedback parameters", open=False):
|
214 |
+
feedback_start = gr.Slider(0.0, 1.0, value=0.0, label="Feedback start", info="Fraction of denoising steps starting from which to use max. feedback weight.")
|
215 |
+
feedback_end = gr.Slider(0.0, 1.0, value=0.8, label="Feedback end", info="Up to what fraction of denoising steps to use max. feedback weight.")
|
216 |
+
feedback_min_weight = gr.Slider(0.0, 1.0, value=0.0, label="Feedback min. weight", info="Attention weight of feedback images when turned off (set to 0.0 to disable)")
|
217 |
+
feedback_max_weight = gr.Slider(0.0, 1.0, value=0.8, label="Feedback max. weight", info="Attention weight of feedback images when turned on (set to 0.0 to disable)")
|
218 |
+
feedback_neg_scale = gr.Slider(0.0, 10.0, value=0.5, label="Neg. feedback scale", info="Attention weight of disliked images relative to liked images (set to 0.0 to disable negative feedback)")
|
219 |
+
|
220 |
+
with gr.Column():
|
221 |
+
gallery = gr.Gallery(label="Generated images")
|
222 |
+
|
223 |
+
like_btns = []
|
224 |
+
dislike_btns = []
|
225 |
+
with gr.Row():
|
226 |
+
for i in range(0, 2):
|
227 |
+
like_btn = gr.Button(f"π Image {i+1}", elem_classes="btn-green")
|
228 |
+
like_btns.append(like_btn)
|
229 |
+
with gr.Row():
|
230 |
+
for i in range(2, 4):
|
231 |
+
like_btn = gr.Button(f"π Image {i+1}", elem_classes="btn-green")
|
232 |
+
like_btns.append(like_btn)
|
233 |
+
with gr.Row():
|
234 |
+
for i in range(0, 2):
|
235 |
+
dislike_btn = gr.Button(f"π Image {i+1}", elem_classes="btn-red")
|
236 |
+
dislike_btns.append(dislike_btn)
|
237 |
+
with gr.Row():
|
238 |
+
for i in range(2, 4):
|
239 |
+
dislike_btn = gr.Button(f"π Image {i+1}", elem_classes="btn-red")
|
240 |
+
dislike_btns.append(dislike_btn)
|
241 |
+
|
242 |
+
prev_seed = gr.Number(-1, label="Previous seed", interactive=False)
|
243 |
+
prev_seed_hid = gr.Number(-1, visible=False)
|
244 |
+
|
245 |
+
generate_params = [
|
246 |
+
feedback_enabled,
|
247 |
+
max_feedback_imgs,
|
248 |
+
prompt,
|
249 |
+
neg_prompt,
|
250 |
+
liked_imgs,
|
251 |
+
disliked_imgs,
|
252 |
+
denoising_steps,
|
253 |
+
guidance_scale,
|
254 |
+
feedback_start,
|
255 |
+
feedback_end,
|
256 |
+
feedback_min_weight,
|
257 |
+
feedback_max_weight,
|
258 |
+
feedback_neg_scale,
|
259 |
+
batch_size,
|
260 |
+
seed,
|
261 |
+
]
|
262 |
+
submit_btn.click(generate_fn, generate_params, [gallery, curr_imgs, prev_seed_hid], queue=True)
|
263 |
+
prev_seed_hid.change(duplicate_seed_value, prev_seed_hid, prev_seed, queue=False)
|
264 |
+
|
265 |
+
for i, like_btn in enumerate(like_btns):
|
266 |
+
like_btn.click(functools.partial(add_img_from_list, i), [curr_imgs, liked_imgs], [like_gallery, liked_imgs], queue=False)
|
267 |
+
for i, dislike_btn in enumerate(dislike_btns):
|
268 |
+
dislike_btn.click(functools.partial(add_img_from_list, i), [curr_imgs, disliked_imgs], [dislike_gallery, disliked_imgs], queue=False)
|
269 |
+
|
270 |
+
like_gallery.select(remove_img_from_list, [liked_imgs], [like_gallery, liked_imgs], queue=False)
|
271 |
+
dislike_gallery.select(remove_img_from_list, [disliked_imgs], [dislike_gallery, disliked_imgs], queue=False)
|
272 |
+
|
273 |
+
liked_img_input.upload(add_img, [liked_img_input, liked_imgs], [liked_img_input, like_gallery, liked_imgs], queue=False)
|
274 |
+
disliked_img_input.upload(add_img, [disliked_img_input, disliked_imgs], [disliked_img_input, dislike_gallery, disliked_imgs], queue=False)
|
275 |
+
|
276 |
+
clear_liked_btn.click(lambda: [[], []], None, [liked_imgs, like_gallery], queue=False)
|
277 |
+
clear_disliked_btn.click(lambda: [[], []], None, [disliked_imgs, dislike_gallery], queue=False)
|
278 |
+
|
279 |
+
demo.queue()
|
280 |
demo.launch(debug=True)
|