fastdiffusion.bib

@inproceedings{van_den_oord_neural_2017,
	title = {Neural Discrete Representation Learning},
	volume = {30},
	url = {https://papers.nips.cc/paper/2017/hash/7a98af17e63a0ac09ce2e96d03992fbc-Abstract.html},
	abstract = {Learning useful representations without supervision remains a key challenge in machine learning. In this paper, we propose a simple yet powerful generative model that learns such discrete representations. Our model, the Vector Quantised-Variational {AutoEncoder} ({VQ}-{VAE}), differs from {VAEs} in two key ways: the encoder network outputs discrete, rather than continuous, codes; and the prior is learnt rather than static. In order to learn a discrete latent representation, we incorporate ideas from vector quantisation ({VQ}). Using the {VQ} method allows the model to circumvent issues of ``posterior collapse'' -— where the latents are ignored when they are paired with a powerful autoregressive decoder -— typically observed in the {VAE} framework. Pairing these representations with an autoregressive prior, the model can generate high quality images, videos, and speech as well as doing high quality speaker conversion and unsupervised learning of phonemes, providing further evidence of the utility of the learnt representations.},
	booktitle = {Advances in Neural Information Processing Systems},
	publisher = {Curran Associates, Inc.},
	author = {van den Oord, Aaron and Vinyals, Oriol and kavukcuoglu, koray},
	urldate = {2022-09-21},
	date = {2017},
	file = {Full Text PDF:/Users/seem/Zotero/storage/PMYFBS83/van den Oord et al. - 2017 - Neural Discrete Representation Learning.pdf:application/pdf},
}

@misc{tolstikhin_mlp-mixer_2021,
	title = {{MLP}-Mixer: An all-{MLP} Architecture for Vision},
	url = {http://arxiv.org/abs/2105.01601},
	doi = {10.48550/arXiv.2105.01601},
	shorttitle = {{MLP}-Mixer},
	abstract = {Convolutional Neural Networks ({CNNs}) are the go-to model for computer vision. Recently, attention-based networks, such as the Vision Transformer, have also become popular. In this paper we show that while convolutions and attention are both sufficient for good performance, neither of them are necessary. We present {MLP}-Mixer, an architecture based exclusively on multi-layer perceptrons ({MLPs}). {MLP}-Mixer contains two types of layers: one with {MLPs} applied independently to image patches (i.e. "mixing" the per-location features), and one with {MLPs} applied across patches (i.e. "mixing" spatial information). When trained on large datasets, or with modern regularization schemes, {MLP}-Mixer attains competitive scores on image classification benchmarks, with pre-training and inference cost comparable to state-of-the-art models. We hope that these results spark further research beyond the realms of well established {CNNs} and Transformers.},
	number = {{arXiv}:2105.01601},
	publisher = {{arXiv}},
	author = {Tolstikhin, Ilya and Houlsby, Neil and Kolesnikov, Alexander and Beyer, Lucas and Zhai, Xiaohua and Unterthiner, Thomas and Yung, Jessica and Steiner, Andreas and Keysers, Daniel and Uszkoreit, Jakob and Lucic, Mario and Dosovitskiy, Alexey},
	urldate = {2022-09-22},
	date = {2021-06-11},
	eprinttype = {arxiv},
	eprint = {2105.01601 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/UAG7Q9WT/Tolstikhin et al. - 2021 - MLP-Mixer An all-MLP Architecture for Vision.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/AFMSXK9X/2105.html:text/html},
}

@misc{nichol_glide_2022,
	title = {{GLIDE}: Towards Photorealistic Image Generation and Editing with Text-Guided Diffusion Models},
	url = {http://arxiv.org/abs/2112.10741},
	doi = {10.48550/arXiv.2112.10741},
	shorttitle = {{GLIDE}},
	abstract = {Diffusion models have recently been shown to generate high-quality synthetic images, especially when paired with a guidance technique to trade off diversity for fidelity. We explore diffusion models for the problem of text-conditional image synthesis and compare two different guidance strategies: {CLIP} guidance and classifier-free guidance. We find that the latter is preferred by human evaluators for both photorealism and caption similarity, and often produces photorealistic samples. Samples from a 3.5 billion parameter text-conditional diffusion model using classifier-free guidance are favored by human evaluators to those from {DALL}-E, even when the latter uses expensive {CLIP} reranking. Additionally, we find that our models can be fine-tuned to perform image inpainting, enabling powerful text-driven image editing. We train a smaller model on a filtered dataset and release the code and weights at https://github.com/openai/glide-text2im.},
	number = {{arXiv}:2112.10741},
	publisher = {{arXiv}},
	author = {Nichol, Alex and Dhariwal, Prafulla and Ramesh, Aditya and Shyam, Pranav and Mishkin, Pamela and {McGrew}, Bob and Sutskever, Ilya and Chen, Mark},
	urldate = {2022-09-22},
	date = {2022-03-08},
	eprinttype = {arxiv},
	eprint = {2112.10741 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Graphics},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/XSG377JU/Nichol et al. - 2022 - GLIDE Towards Photorealistic Image Generation and.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/TVNKG8TY/2112.html:text/html},
}

@misc{furst_cloob_2022,
	title = {{CLOOB}: Modern Hopfield Networks with {InfoLOOB} Outperform {CLIP}},
	url = {http://arxiv.org/abs/2110.11316},
	doi = {10.48550/arXiv.2110.11316},
	shorttitle = {{CLOOB}},
	abstract = {{CLIP} yielded impressive results on zero-shot transfer learning tasks and is considered as a foundation model like {BERT} or {GPT}3. {CLIP} vision models that have a rich representation are pre-trained using the {InfoNCE} objective and natural language supervision before they are fine-tuned on particular tasks. Though {CLIP} excels at zero-shot transfer learning, it suffers from an explaining away problem, that is, it focuses on one or few features, while neglecting other relevant features. This problem is caused by insufficiently extracting the covariance structure in the original multi-modal data. We suggest to use modern Hopfield networks to tackle the problem of explaining away. Their retrieved embeddings have an enriched covariance structure derived from co-occurrences of features in the stored embeddings. However, modern Hopfield networks increase the saturation effect of the {InfoNCE} objective which hampers learning. We propose to use the {InfoLOOB} objective to mitigate this saturation effect. We introduce the novel ``Contrastive Leave One Out Boost'' ({CLOOB}), which uses modern Hopfield networks for covariance enrichment together with the {InfoLOOB} objective. In experiments we compare {CLOOB} to {CLIP} after pre-training on the Conceptual Captions and the {YFCC} dataset with respect to their zero-shot transfer learning performance on other datasets. {CLOOB} consistently outperforms {CLIP} at zero-shot transfer learning across all considered architectures and datasets.},
	number = {{arXiv}:2110.11316},
	publisher = {{arXiv}},
	author = {Fürst, Andreas and Rumetshofer, Elisabeth and Lehner, Johannes and Tran, Viet and Tang, Fei and Ramsauer, Hubert and Kreil, David and Kopp, Michael and Klambauer, Günter and Bitto-Nemling, Angela and Hochreiter, Sepp},
	urldate = {2022-09-22},
	date = {2022-06-13},
	eprinttype = {arxiv},
	eprint = {2110.11316 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/ZTQ99TU8/Fürst et al. - 2022 - CLOOB Modern Hopfield Networks with InfoLOOB Outp.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/32JV45N4/2110.html:text/html},
}

@misc{kong_diffwave_2021,
	title = {{DiffWave}: A Versatile Diffusion Model for Audio Synthesis},
	url = {http://arxiv.org/abs/2009.09761},
	doi = {10.48550/arXiv.2009.09761},
	shorttitle = {{DiffWave}},
	abstract = {In this work, we propose {DiffWave}, a versatile diffusion probabilistic model for conditional and unconditional waveform generation. The model is non-autoregressive, and converts the white noise signal into structured waveform through a Markov chain with a constant number of steps at synthesis. It is efficiently trained by optimizing a variant of variational bound on the data likelihood. {DiffWave} produces high-fidelity audios in different waveform generation tasks, including neural vocoding conditioned on mel spectrogram, class-conditional generation, and unconditional generation. We demonstrate that {DiffWave} matches a strong {WaveNet} vocoder in terms of speech quality ({MOS}: 4.44 versus 4.43), while synthesizing orders of magnitude faster. In particular, it significantly outperforms autoregressive and {GAN}-based waveform models in the challenging unconditional generation task in terms of audio quality and sample diversity from various automatic and human evaluations.},
	number = {{arXiv}:2009.09761},
	publisher = {{arXiv}},
	author = {Kong, Zhifeng and Ping, Wei and Huang, Jiaji and Zhao, Kexin and Catanzaro, Bryan},
	urldate = {2022-09-22},
	date = {2021-03-30},
	eprinttype = {arxiv},
	eprint = {2009.09761 [cs, eess, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Computation and Language, Computer Science - Sound, Electrical Engineering and Systems Science - Audio and Speech Processing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/G5HAIJ52/Kong et al. - 2021 - DiffWave A Versatile Diffusion Model for Audio Sy.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/ERYV5PZG/2009.html:text/html},
}

@misc{gu_vector_2022,
	title = {Vector Quantized Diffusion Model for Text-to-Image Synthesis},
	url = {http://arxiv.org/abs/2111.14822},
	abstract = {We present the vector quantized diffusion ({VQ}-Diffusion) model for text-to-image generation. This method is based on a vector quantized variational autoencoder ({VQ}-{VAE}) whose latent space is modeled by a conditional variant of the recently developed Denoising Diffusion Probabilistic Model ({DDPM}). We find that this latent-space method is well-suited for text-to-image generation tasks because it not only eliminates the unidirectional bias with existing methods but also allows us to incorporate a mask-and-replace diffusion strategy to avoid the accumulation of errors, which is a serious problem with existing methods. Our experiments show that the {VQ}-Diffusion produces significantly better text-to-image generation results when compared with conventional autoregressive ({AR}) models with similar numbers of parameters. Compared with previous {GAN}-based text-to-image methods, our {VQ}-Diffusion can handle more complex scenes and improve the synthesized image quality by a large margin. Finally, we show that the image generation computation in our method can be made highly efficient by reparameterization. With traditional {AR} methods, the text-to-image generation time increases linearly with the output image resolution and hence is quite time consuming even for normal size images. The {VQ}-Diffusion allows us to achieve a better trade-off between quality and speed. Our experiments indicate that the {VQ}-Diffusion model with the reparameterization is fifteen times faster than traditional {AR} methods while achieving a better image quality.},
	number = {{arXiv}:2111.14822},
	publisher = {{arXiv}},
	author = {Gu, Shuyang and Chen, Dong and Bao, Jianmin and Wen, Fang and Zhang, Bo and Chen, Dongdong and Yuan, Lu and Guo, Baining},
	urldate = {2022-09-22},
	date = {2022-03-03},
	eprinttype = {arxiv},
	eprint = {2111.14822 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/TEEPHIPM/Gu et al. - 2022 - Vector Quantized Diffusion Model for Text-to-Image.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/B7U7XQ7S/2111.html:text/html},
}

@misc{tang_improved_2022,
	title = {Improved Vector Quantized Diffusion Models},
	url = {http://arxiv.org/abs/2205.16007},
	abstract = {Vector quantized diffusion ({VQ}-Diffusion) is a powerful generative model for text-to-image synthesis, but sometimes can still generate low-quality samples or weakly correlated images with text input. We find these issues are mainly due to the flawed sampling strategy. In this paper, we propose two important techniques to further improve the sample quality of {VQ}-Diffusion. 1) We explore classifier-free guidance sampling for discrete denoising diffusion model and propose a more general and effective implementation of classifier-free guidance. 2) We present a high-quality inference strategy to alleviate the joint distribution issue in {VQ}-Diffusion. Finally, we conduct experiments on various datasets to validate their effectiveness and show that the improved {VQ}-Diffusion suppresses the vanilla version by large margins. We achieve an 8.44 {FID} score on {MSCOCO}, surpassing {VQ}-Diffusion by 5.42 {FID} score. When trained on {ImageNet}, we dramatically improve the {FID} score from 11.89 to 4.83, demonstrating the superiority of our proposed techniques.},
	number = {{arXiv}:2205.16007},
	publisher = {{arXiv}},
	author = {Tang, Zhicong and Gu, Shuyang and Bao, Jianmin and Chen, Dong and Wen, Fang},
	urldate = {2022-09-22},
	date = {2022-05-31},
	eprinttype = {arxiv},
	eprint = {2205.16007 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/YYKZFYNE/Tang et al. - 2022 - Improved Vector Quantized Diffusion Models.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/2VP2PTFC/2205.html:text/html},
}

@misc{borsos_audiolm_2022,
	title = {{AudioLM}: a Language Modeling Approach to Audio Generation},
	url = {http://arxiv.org/abs/2209.03143},
	doi = {10.48550/arXiv.2209.03143},
	shorttitle = {{AudioLM}},
	abstract = {We introduce {AudioLM}, a framework for high-quality audio generation with long-term consistency. {AudioLM} maps the input audio to a sequence of discrete tokens and casts audio generation as a language modeling task in this representation space. We show how existing audio tokenizers provide different trade-offs between reconstruction quality and long-term structure, and we propose a hybrid tokenization scheme to achieve both objectives. Namely, we leverage the discretized activations of a masked language model pre-trained on audio to capture long-term structure and the discrete codes produced by a neural audio codec to achieve high-quality synthesis. By training on large corpora of raw audio waveforms, {AudioLM} learns to generate natural and coherent continuations given short prompts. When trained on speech, and without any transcript or annotation, {AudioLM} generates syntactically and semantically plausible speech continuations while also maintaining speaker identity and prosody for unseen speakers. Furthermore, we demonstrate how our approach extends beyond speech by generating coherent piano music continuations, despite being trained without any symbolic representation of music.},
	number = {{arXiv}:2209.03143},
	publisher = {{arXiv}},
	author = {Borsos, Zalán and Marinier, Raphaël and Vincent, Damien and Kharitonov, Eugene and Pietquin, Olivier and Sharifi, Matt and Teboul, Olivier and Grangier, David and Tagliasacchi, Marco and Zeghidour, Neil},
	urldate = {2022-09-22},
	date = {2022-09-07},
	eprinttype = {arxiv},
	eprint = {2209.03143 [cs, eess]},
	keywords = {Computer Science - Machine Learning, Computer Science - Sound, Electrical Engineering and Systems Science - Audio and Speech Processing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/ES53PJFE/Borsos et al. - 2022 - AudioLM a Language Modeling Approach to Audio Gen.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/ASEMDHKH/2209.html:text/html},
}

@misc{xiao_tackling_2022,
	title = {Tackling the Generative Learning Trilemma with Denoising Diffusion {GANs}},
	url = {http://arxiv.org/abs/2112.07804},
	doi = {10.48550/arXiv.2112.07804},
	abstract = {A wide variety of deep generative models has been developed in the past decade. Yet, these models often struggle with simultaneously addressing three key requirements including: high sample quality, mode coverage, and fast sampling. We call the challenge imposed by these requirements the generative learning trilemma, as the existing models often trade some of them for others. Particularly, denoising diffusion models have shown impressive sample quality and diversity, but their expensive sampling does not yet allow them to be applied in many real-world applications. In this paper, we argue that slow sampling in these models is fundamentally attributed to the Gaussian assumption in the denoising step which is justified only for small step sizes. To enable denoising with large steps, and hence, to reduce the total number of denoising steps, we propose to model the denoising distribution using a complex multimodal distribution. We introduce denoising diffusion generative adversarial networks (denoising diffusion {GANs}) that model each denoising step using a multimodal conditional {GAN}. Through extensive evaluations, we show that denoising diffusion {GANs} obtain sample quality and diversity competitive with original diffusion models while being 2000\${\textbackslash}times\$ faster on the {CIFAR}-10 dataset. Compared to traditional {GANs}, our model exhibits better mode coverage and sample diversity. To the best of our knowledge, denoising diffusion {GAN} is the first model that reduces sampling cost in diffusion models to an extent that allows them to be applied to real-world applications inexpensively. Project page and code can be found at https://nvlabs.github.io/denoising-diffusion-gan},
	number = {{arXiv}:2112.07804},
	publisher = {{arXiv}},
	author = {Xiao, Zhisheng and Kreis, Karsten and Vahdat, Arash},
	urldate = {2022-09-22},
	date = {2022-04-04},
	eprinttype = {arxiv},
	eprint = {2112.07804 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/ZKNFUEJ2/Xiao et al. - 2022 - Tackling the Generative Learning Trilemma with Den.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/76QDQN4S/2112.html:text/html},
}

@misc{ho_classifier-free_2022,
	title = {Classifier-Free Diffusion Guidance},
	url = {http://arxiv.org/abs/2207.12598},
	doi = {10.48550/arXiv.2207.12598},
	abstract = {Classifier guidance is a recently introduced method to trade off mode coverage and sample fidelity in conditional diffusion models post training, in the same spirit as low temperature sampling or truncation in other types of generative models. Classifier guidance combines the score estimate of a diffusion model with the gradient of an image classifier and thereby requires training an image classifier separate from the diffusion model. It also raises the question of whether guidance can be performed without a classifier. We show that guidance can be indeed performed by a pure generative model without such a classifier: in what we call classifier-free guidance, we jointly train a conditional and an unconditional diffusion model, and we combine the resulting conditional and unconditional score estimates to attain a trade-off between sample quality and diversity similar to that obtained using classifier guidance.},
	number = {{arXiv}:2207.12598},
	publisher = {{arXiv}},
	author = {Ho, Jonathan and Salimans, Tim},
	urldate = {2022-09-22},
	date = {2022-07-25},
	eprinttype = {arxiv},
	eprint = {2207.12598 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/RY5PGTZH/Ho and Salimans - 2022 - Classifier-Free Diffusion Guidance.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/HT3QDDWM/2207.html:text/html},
}

@misc{daras_soft_2022,
	title = {Soft Diffusion: Score Matching for General Corruptions},
	url = {http://arxiv.org/abs/2209.05442},
	doi = {10.48550/arXiv.2209.05442},
	shorttitle = {Soft Diffusion},
	abstract = {We define a broader family of corruption processes that generalizes previously known diffusion models. To reverse these general diffusions, we propose a new objective called Soft Score Matching that provably learns the score function for any linear corruption process and yields state of the art results for {CelebA}. Soft Score Matching incorporates the degradation process in the network and trains the model to predict a clean image that after corruption matches the diffused observation. We show that our objective learns the gradient of the likelihood under suitable regularity conditions for the family of corruption processes. We further develop a principled way to select the corruption levels for general diffusion processes and a novel sampling method that we call Momentum Sampler. We evaluate our framework with the corruption being Gaussian Blur and low magnitude additive noise. Our method achieves state-of-the-art {FID} score \$1.85\$ on {CelebA}-64, outperforming all previous linear diffusion models. We also show significant computational benefits compared to vanilla denoising diffusion.},
	number = {{arXiv}:2209.05442},
	publisher = {{arXiv}},
	author = {Daras, Giannis and Delbracio, Mauricio and Talebi, Hossein and Dimakis, Alexandros G. and Milanfar, Peyman},
	urldate = {2022-09-21},
	date = {2022-09-12},
	eprinttype = {arxiv},
	eprint = {2209.05442 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/EKI8R8DP/Daras et al. - 2022 - Soft Diffusion Score Matching for General Corrupt.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/9YD9MKRZ/2209.html:text/html},
}

@misc{huang_normalization_2020,
	title = {Normalization Techniques in Training {DNNs}: Methodology, Analysis and Application},
	url = {http://arxiv.org/abs/2009.12836},
	shorttitle = {Normalization Techniques in Training {DNNs}},
	abstract = {Normalization techniques are essential for accelerating the training and improving the generalization of deep neural networks ({DNNs}), and have successfully been used in various applications. This paper reviews and comments on the past, present and future of normalization methods in the context of {DNN} training. We provide a unified picture of the main motivation behind different approaches from the perspective of optimization, and present a taxonomy for understanding the similarities and differences between them. Specifically, we decompose the pipeline of the most representative normalizing activation methods into three components: the normalization area partitioning, normalization operation and normalization representation recovery. In doing so, we provide insight for designing new normalization technique. Finally, we discuss the current progress in understanding normalization methods, and provide a comprehensive review of the applications of normalization for particular tasks, in which it can effectively solve the key issues.},
	number = {{arXiv}:2009.12836},
	publisher = {{arXiv}},
	author = {Huang, Lei and Qin, Jie and Zhou, Yi and Zhu, Fan and Liu, Li and Shao, Ling},
	urldate = {2022-09-21},
	date = {2020-09-27},
	eprinttype = {arxiv},
	eprint = {2009.12836 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/4RUV4D93/Huang et al. - 2020 - Normalization Techniques in Training DNNs Methodo.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/Z6AAK4AB/2009.html:text/html},
}

@misc{luhman_improving_2022,
	title = {Improving Diffusion Model Efficiency Through Patching},
	url = {http://arxiv.org/abs/2207.04316},
	abstract = {Diffusion models are a powerful class of generative models that iteratively denoise samples to produce data. While many works have focused on the number of iterations in this sampling procedure, few have focused on the cost of each iteration. We find that adding a simple {ViT}-style patching transformation can considerably reduce a diffusion model's sampling time and memory usage. We justify our approach both through an analysis of the diffusion model objective, and through empirical experiments on {LSUN} Church, {ImageNet} 256, and {FFHQ} 1024. We provide implementations in Tensorflow and Pytorch.},
	number = {{arXiv}:2207.04316},
	publisher = {{arXiv}},
	author = {Luhman, Troy and Luhman, Eric},
	urldate = {2022-09-21},
	date = {2022-07-09},
	eprinttype = {arxiv},
	eprint = {2207.04316 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/BSGZAEBZ/Luhman and Luhman - 2022 - Improving Diffusion Model Efficiency Through Patch.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/KK55UFKE/2207.html:text/html},
}

@misc{esser_taming_2021,
	title = {Taming Transformers for High-Resolution Image Synthesis},
	url = {http://arxiv.org/abs/2012.09841},
	abstract = {Designed to learn long-range interactions on sequential data, transformers continue to show state-of-the-art results on a wide variety of tasks. In contrast to {CNNs}, they contain no inductive bias that prioritizes local interactions. This makes them expressive, but also computationally infeasible for long sequences, such as high-resolution images. We demonstrate how combining the effectiveness of the inductive bias of {CNNs} with the expressivity of transformers enables them to model and thereby synthesize high-resolution images. We show how to (i) use {CNNs} to learn a context-rich vocabulary of image constituents, and in turn (ii) utilize transformers to efficiently model their composition within high-resolution images. Our approach is readily applied to conditional synthesis tasks, where both non-spatial information, such as object classes, and spatial information, such as segmentations, can control the generated image. In particular, we present the first results on semantically-guided synthesis of megapixel images with transformers and obtain the state of the art among autoregressive models on class-conditional {ImageNet}. Code and pretrained models can be found at https://github.com/{CompVis}/taming-transformers .},
	number = {{arXiv}:2012.09841},
	publisher = {{arXiv}},
	author = {Esser, Patrick and Rombach, Robin and Ommer, Björn},
	urldate = {2022-09-21},
	date = {2021-06-23},
	eprinttype = {arxiv},
	eprint = {2012.09841 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/WA5I23A7/Esser et al. - 2021 - Taming Transformers for High-Resolution Image Synt.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/65GLYB3W/2012.html:text/html},
}

@misc{choi_perception_2022,
	title = {Perception Prioritized Training of Diffusion Models},
	url = {http://arxiv.org/abs/2204.00227},
	abstract = {Diffusion models learn to restore noisy data, which is corrupted with different levels of noise, by optimizing the weighted sum of the corresponding loss terms, i.e., denoising score matching loss. In this paper, we show that restoring data corrupted with certain noise levels offers a proper pretext task for the model to learn rich visual concepts. We propose to prioritize such noise levels over other levels during training, by redesigning the weighting scheme of the objective function. We show that our simple redesign of the weighting scheme significantly improves the performance of diffusion models regardless of the datasets, architectures, and sampling strategies.},
	number = {{arXiv}:2204.00227},
	publisher = {{arXiv}},
	author = {Choi, Jooyoung and Lee, Jungbeom and Shin, Chaehun and Kim, Sungwon and Kim, Hyunwoo and Yoon, Sungroh},
	urldate = {2022-09-21},
	date = {2022-04-01},
	eprinttype = {arxiv},
	eprint = {2204.00227 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/YDRS5E6G/Choi et al. - 2022 - Perception Prioritized Training of Diffusion Model.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/ZWP83CK4/2204.html:text/html},
}

@misc{rombach_high-resolution_2022,
	title = {High-Resolution Image Synthesis with Latent Diffusion Models},
	url = {http://arxiv.org/abs/2112.10752},
	abstract = {By decomposing the image formation process into a sequential application of denoising autoencoders, diffusion models ({DMs}) achieve state-of-the-art synthesis results on image data and beyond. Additionally, their formulation allows for a guiding mechanism to control the image generation process without retraining. However, since these models typically operate directly in pixel space, optimization of powerful {DMs} often consumes hundreds of {GPU} days and inference is expensive due to sequential evaluations. To enable {DM} training on limited computational resources while retaining their quality and flexibility, we apply them in the latent space of powerful pretrained autoencoders. In contrast to previous work, training diffusion models on such a representation allows for the first time to reach a near-optimal point between complexity reduction and detail preservation, greatly boosting visual fidelity. By introducing cross-attention layers into the model architecture, we turn diffusion models into powerful and flexible generators for general conditioning inputs such as text or bounding boxes and high-resolution synthesis becomes possible in a convolutional manner. Our latent diffusion models ({LDMs}) achieve a new state of the art for image inpainting and highly competitive performance on various tasks, including unconditional image generation, semantic scene synthesis, and super-resolution, while significantly reducing computational requirements compared to pixel-based {DMs}. Code is available at https://github.com/{CompVis}/latent-diffusion .},
	number = {{arXiv}:2112.10752},
	publisher = {{arXiv}},
	author = {Rombach, Robin and Blattmann, Andreas and Lorenz, Dominik and Esser, Patrick and Ommer, Björn},
	urldate = {2022-09-21},
	date = {2022-04-13},
	eprinttype = {arxiv},
	eprint = {2112.10752 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/4FPMLBZR/Rombach et al. - 2022 - High-Resolution Image Synthesis with Latent Diffus.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/5WA9YKHX/2112.html:text/html},
}

@misc{caron_emerging_2021,
	title = {Emerging Properties in Self-Supervised Vision Transformers},
	url = {http://arxiv.org/abs/2104.14294},
	abstract = {In this paper, we question if self-supervised learning provides new properties to Vision Transformer ({ViT}) that stand out compared to convolutional networks (convnets). Beyond the fact that adapting self-supervised methods to this architecture works particularly well, we make the following observations: first, self-supervised {ViT} features contain explicit information about the semantic segmentation of an image, which does not emerge as clearly with supervised {ViTs}, nor with convnets. Second, these features are also excellent k-{NN} classifiers, reaching 78.3\% top-1 on {ImageNet} with a small {ViT}. Our study also underlines the importance of momentum encoder, multi-crop training, and the use of small patches with {ViTs}. We implement our findings into a simple self-supervised method, called {DINO}, which we interpret as a form of self-distillation with no labels. We show the synergy between {DINO} and {ViTs} by achieving 80.1\% top-1 on {ImageNet} in linear evaluation with {ViT}-Base.},
	number = {{arXiv}:2104.14294},
	publisher = {{arXiv}},
	author = {Caron, Mathilde and Touvron, Hugo and Misra, Ishan and Jégou, Hervé and Mairal, Julien and Bojanowski, Piotr and Joulin, Armand},
	urldate = {2022-09-21},
	date = {2021-05-24},
	eprinttype = {arxiv},
	eprint = {2104.14294 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/5NGTHJ2K/Caron et al. - 2021 - Emerging Properties in Self-Supervised Vision Tran.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/KZCXD57C/2104.html:text/html},
}

@misc{karras_elucidating_2022,
	title = {Elucidating the Design Space of Diffusion-Based Generative Models},
	url = {http://arxiv.org/abs/2206.00364},
	abstract = {We argue that the theory and practice of diffusion-based generative models are currently unnecessarily convoluted and seek to remedy the situation by presenting a design space that clearly separates the concrete design choices. This lets us identify several changes to both the sampling and training processes, as well as preconditioning of the score networks. Together, our improvements yield new state-of-the-art {FID} of 1.79 for {CIFAR}-10 in a class-conditional setting and 1.97 in an unconditional setting, with much faster sampling (35 network evaluations per image) than prior designs. To further demonstrate their modular nature, we show that our design changes dramatically improve both the efficiency and quality obtainable with pre-trained score networks from previous work, including improving the {FID} of an existing {ImageNet}-64 model from 2.07 to near-{SOTA} 1.55.},
	number = {{arXiv}:2206.00364},
	publisher = {{arXiv}},
	author = {Karras, Tero and Aittala, Miika and Aila, Timo and Laine, Samuli},
	urldate = {2022-09-21},
	date = {2022-06-01},
	eprinttype = {arxiv},
	eprint = {2206.00364 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Neural and Evolutionary Computing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/V9JW5GQF/Karras et al. - 2022 - Elucidating the Design Space of Diffusion-Based Ge.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/XF8AW72H/2206.html:text/html},
}

@misc{ho_denoising_2020,
	title = {Denoising Diffusion Probabilistic Models},
	url = {http://arxiv.org/abs/2006.11239},
	abstract = {We present high quality image synthesis results using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics. Our best results are obtained by training on a weighted variational bound designed according to a novel connection between diffusion probabilistic models and denoising score matching with Langevin dynamics, and our models naturally admit a progressive lossy decompression scheme that can be interpreted as a generalization of autoregressive decoding. On the unconditional {CIFAR}10 dataset, we obtain an Inception score of 9.46 and a state-of-the-art {FID} score of 3.17. On 256x256 {LSUN}, we obtain sample quality similar to {ProgressiveGAN}. Our implementation is available at https://github.com/hojonathanho/diffusion},
	number = {{arXiv}:2006.11239},
	publisher = {{arXiv}},
	author = {Ho, Jonathan and Jain, Ajay and Abbeel, Pieter},
	urldate = {2022-09-21},
	date = {2020-12-16},
	eprinttype = {arxiv},
	eprint = {2006.11239 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/WX3MINXE/Ho et al. - 2020 - Denoising Diffusion Probabilistic Models.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/8KIPJSI9/2006.html:text/html},
}

@misc{song_denoising_2022,
	title = {Denoising Diffusion Implicit Models},
	url = {http://arxiv.org/abs/2010.02502},
	abstract = {Denoising diffusion probabilistic models ({DDPMs}) have achieved high quality image generation without adversarial training, yet they require simulating a Markov chain for many steps to produce a sample. To accelerate sampling, we present denoising diffusion implicit models ({DDIMs}), a more efficient class of iterative implicit probabilistic models with the same training procedure as {DDPMs}. In {DDPMs}, the generative process is defined as the reverse of a Markovian diffusion process. We construct a class of non-Markovian diffusion processes that lead to the same training objective, but whose reverse process can be much faster to sample from. We empirically demonstrate that {DDIMs} can produce high quality samples \$10 {\textbackslash}times\$ to \$50 {\textbackslash}times\$ faster in terms of wall-clock time compared to {DDPMs}, allow us to trade off computation for sample quality, and can perform semantically meaningful image interpolation directly in the latent space.},
	number = {{arXiv}:2010.02502},
	publisher = {{arXiv}},
	author = {Song, Jiaming and Meng, Chenlin and Ermon, Stefano},
	urldate = {2022-09-21},
	date = {2022-06-09},
	eprinttype = {arxiv},
	eprint = {2010.02502 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/96CGZGGC/Song et al. - 2022 - Denoising Diffusion Implicit Models.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/XCYX42YQ/2010.html:text/html},
}

@misc{bansal_cold_2022,
	title = {Cold Diffusion: Inverting Arbitrary Image Transforms Without Noise},
	url = {http://arxiv.org/abs/2208.09392},
	shorttitle = {Cold Diffusion},
	abstract = {Standard diffusion models involve an image transform -- adding Gaussian noise -- and an image restoration operator that inverts this degradation. We observe that the generative behavior of diffusion models is not strongly dependent on the choice of image degradation, and in fact an entire family of generative models can be constructed by varying this choice. Even when using completely deterministic degradations (e.g., blur, masking, and more), the training and test-time update rules that underlie diffusion models can be easily generalized to create generative models. The success of these fully deterministic models calls into question the community's understanding of diffusion models, which relies on noise in either gradient Langevin dynamics or variational inference, and paves the way for generalized diffusion models that invert arbitrary processes. Our code is available at https://github.com/arpitbansal297/Cold-Diffusion-Models},
	number = {{arXiv}:2208.09392},
	publisher = {{arXiv}},
	author = {Bansal, Arpit and Borgnia, Eitan and Chu, Hong-Min and Li, Jie S. and Kazemi, Hamid and Huang, Furong and Goldblum, Micah and Geiping, Jonas and Goldstein, Tom},
	urldate = {2022-09-21},
	date = {2022-08-19},
	eprinttype = {arxiv},
	eprint = {2208.09392 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/TIGIVMJT/Bansal et al. - 2022 - Cold Diffusion Inverting Arbitrary Image Transfor.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/M2B4QAYR/2208.html:text/html},
}

@misc{karras_training_2020,
	title = {Training Generative Adversarial Networks with Limited Data},
	url = {http://arxiv.org/abs/2006.06676},
	doi = {10.48550/arXiv.2006.06676},
	abstract = {Training generative adversarial networks ({GAN}) using too little data typically leads to discriminator overfitting, causing training to diverge. We propose an adaptive discriminator augmentation mechanism that significantly stabilizes training in limited data regimes. The approach does not require changes to loss functions or network architectures, and is applicable both when training from scratch and when fine-tuning an existing {GAN} on another dataset. We demonstrate, on several datasets, that good results are now possible using only a few thousand training images, often matching {StyleGAN}2 results with an order of magnitude fewer images. We expect this to open up new application domains for {GANs}. We also find that the widely used {CIFAR}-10 is, in fact, a limited data benchmark, and improve the record {FID} from 5.59 to 2.42.},
	number = {{arXiv}:2006.06676},
	publisher = {{arXiv}},
	author = {Karras, Tero and Aittala, Miika and Hellsten, Janne and Laine, Samuli and Lehtinen, Jaakko and Aila, Timo},
	urldate = {2022-09-24},
	date = {2020-10-07},
	eprinttype = {arxiv},
	eprint = {2006.06676 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Neural and Evolutionary Computing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/7ILWU5YS/Karras et al. - 2020 - Training Generative Adversarial Networks with Limi.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/CDGCE6KL/2006.html:text/html},
}

@misc{yang_vtoonify_2022,
	title = {{VToonify}: Controllable High-Resolution Portrait Video Style Transfer},
	url = {http://arxiv.org/abs/2209.11224},
	doi = {10.48550/arXiv.2209.11224},
	shorttitle = {{VToonify}},
	abstract = {Generating high-quality artistic portrait videos is an important and desirable task in computer graphics and vision. Although a series of successful portrait image toonification models built upon the powerful {StyleGAN} have been proposed, these image-oriented methods have obvious limitations when applied to videos, such as the fixed frame size, the requirement of face alignment, missing non-facial details and temporal inconsistency. In this work, we investigate the challenging controllable high-resolution portrait video style transfer by introducing a novel {VToonify} framework. Specifically, {VToonify} leverages the mid- and high-resolution layers of {StyleGAN} to render high-quality artistic portraits based on the multi-scale content features extracted by an encoder to better preserve the frame details. The resulting fully convolutional architecture accepts non-aligned faces in videos of variable size as input, contributing to complete face regions with natural motions in the output. Our framework is compatible with existing {StyleGAN}-based image toonification models to extend them to video toonification, and inherits appealing features of these models for flexible style control on color and intensity. This work presents two instantiations of {VToonify} built upon Toonify and {DualStyleGAN} for collection-based and exemplar-based portrait video style transfer, respectively. Extensive experimental results demonstrate the effectiveness of our proposed {VToonify} framework over existing methods in generating high-quality and temporally-coherent artistic portrait videos with flexible style controls.},
	number = {{arXiv}:2209.11224},
	publisher = {{arXiv}},
	author = {Yang, Shuai and Jiang, Liming and Liu, Ziwei and Loy, Chen Change},
	urldate = {2022-09-26},
	date = {2022-09-23},
	eprinttype = {arxiv},
	eprint = {2209.11224 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Graphics},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/5PKP2UG6/Yang et al. - 2022 - VToonify Controllable High-Resolution Portrait Vi.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/SP5L8IWQ/2209.html:text/html},
}

@misc{watson_learning_2022,
	title = {Learning Fast Samplers for Diffusion Models by Differentiating Through Sample Quality},
	url = {http://arxiv.org/abs/2202.05830},
	doi = {10.48550/arXiv.2202.05830},
	abstract = {Diffusion models have emerged as an expressive family of generative models rivaling {GANs} in sample quality and autoregressive models in likelihood scores. Standard diffusion models typically require hundreds of forward passes through the model to generate a single high-fidelity sample. We introduce Differentiable Diffusion Sampler Search ({DDSS}): a method that optimizes fast samplers for any pre-trained diffusion model by differentiating through sample quality scores. We also present Generalized Gaussian Diffusion Models ({GGDM}), a family of flexible non-Markovian samplers for diffusion models. We show that optimizing the degrees of freedom of {GGDM} samplers by maximizing sample quality scores via gradient descent leads to improved sample quality. Our optimization procedure backpropagates through the sampling process using the reparametrization trick and gradient rematerialization. {DDSS} achieves strong results on unconditional image generation across various datasets (e.g., {FID} scores on {LSUN} church 128x128 of 11.6 with only 10 inference steps, and 4.82 with 20 steps, compared to 51.1 and 14.9 with strongest {DDPM}/{DDIM} baselines). Our method is compatible with any pre-trained diffusion model without fine-tuning or re-training required.},
	number = {{arXiv}:2202.05830},
	publisher = {{arXiv}},
	author = {Watson, Daniel and Chan, William and Ho, Jonathan and Norouzi, Mohammad},
	urldate = {2022-09-26},
	date = {2022-02-11},
	eprinttype = {arxiv},
	eprint = {2202.05830 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/XM5A9FKG/Watson et al. - 2022 - Learning Fast Samplers for Diffusion Models by Dif.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/5Z4B2VMC/2202.html:text/html},
}

@misc{qiao_micro-batch_2020,
	title = {Micro-Batch Training with Batch-Channel Normalization and Weight Standardization},
	url = {http://arxiv.org/abs/1903.10520},
	doi = {10.48550/arXiv.1903.10520},
	abstract = {Batch Normalization ({BN}) has become an out-of-box technique to improve deep network training. However, its effectiveness is limited for micro-batch training, i.e., each {GPU} typically has only 1-2 images for training, which is inevitable for many computer vision tasks, e.g., object detection and semantic segmentation, constrained by memory consumption. To address this issue, we propose Weight Standardization ({WS}) and Batch-Channel Normalization ({BCN}) to bring two success factors of {BN} into micro-batch training: 1) the smoothing effects on the loss landscape and 2) the ability to avoid harmful elimination singularities along the training trajectory. {WS} standardizes the weights in convolutional layers to smooth the loss landscape by reducing the Lipschitz constants of the loss and the gradients; {BCN} combines batch and channel normalizations and leverages estimated statistics of the activations in convolutional layers to keep networks away from elimination singularities. We validate {WS} and {BCN} on comprehensive computer vision tasks, including image classification, object detection, instance segmentation, video recognition and semantic segmentation. All experimental results consistently show that {WS} and {BCN} improve micro-batch training significantly. Moreover, using {WS} and {BCN} with micro-batch training is even able to match or outperform the performances of {BN} with large-batch training.},
	number = {{arXiv}:1903.10520},
	publisher = {{arXiv}},
	author = {Qiao, Siyuan and Wang, Huiyu and Liu, Chenxi and Shen, Wei and Yuille, Alan},
	urldate = {2022-09-26},
	date = {2020-08-09},
	eprinttype = {arxiv},
	eprint = {1903.10520 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/24LEQKUR/Qiao et al. - 2020 - Micro-Batch Training with Batch-Channel Normalizat.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/BDAUZPZJ/1903.html:text/html},
}

@misc{chen_analog_2022,
	title = {Analog Bits: Generating Discrete Data using Diffusion Models with Self-Conditioning},
	url = {http://arxiv.org/abs/2208.04202},
	doi = {10.48550/arXiv.2208.04202},
	shorttitle = {Analog Bits},
	abstract = {We present Bit Diffusion: a simple and generic approach for generating discrete data with continuous diffusion models. The main idea behind our approach is to first represent the discrete data as binary bits, and then train a continuous diffusion model to model these bits as real numbers which we call analog bits. To generate samples, the model first generates the analog bits, which are then thresholded to obtain the bits that represent the discrete variables. We further propose two simple techniques, namely Self-Conditioning and Asymmetric Time Intervals, which lead to a significant improvement in sample quality. Despite its simplicity, the proposed approach can achieve strong performance in both discrete image generation and image captioning tasks. For discrete image generation, we significantly improve previous state-of-the-art on both {CIFAR}-10 (which has 3K discrete 8-bit tokens) and {ImageNet}-64x64 (which has 12K discrete 8-bit tokens), outperforming the best autoregressive model in both sample quality (measured by {FID}) and efficiency. For image captioning on {MS}-{COCO} dataset, our approach achieves competitive results compared to autoregressive models.},
	number = {{arXiv}:2208.04202},
	publisher = {{arXiv}},
	author = {Chen, Ting and Zhang, Ruixiang and Hinton, Geoffrey},
	urldate = {2022-09-26},
	date = {2022-08-08},
	eprinttype = {arxiv},
	eprint = {2208.04202 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/8JX38DM5/Chen et al. - 2022 - Analog Bits Generating Discrete Data using Diffus.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/YZGR8CF4/2208.html:text/html},
}

@misc{liu_pseudo_2022,
	title = {Pseudo Numerical Methods for Diffusion Models on Manifolds},
	url = {http://arxiv.org/abs/2202.09778},
	doi = {10.48550/arXiv.2202.09778},
	abstract = {Denoising Diffusion Probabilistic Models ({DDPMs}) can generate high-quality samples such as image and audio samples. However, {DDPMs} require hundreds to thousands of iterations to produce final samples. Several prior works have successfully accelerated {DDPMs} through adjusting the variance schedule (e.g., Improved Denoising Diffusion Probabilistic Models) or the denoising equation (e.g., Denoising Diffusion Implicit Models ({DDIMs})). However, these acceleration methods cannot maintain the quality of samples and even introduce new noise at a high speedup rate, which limit their practicability. To accelerate the inference process while keeping the sample quality, we provide a fresh perspective that {DDPMs} should be treated as solving differential equations on manifolds. Under such a perspective, we propose pseudo numerical methods for diffusion models ({PNDMs}). Specifically, we figure out how to solve differential equations on manifolds and show that {DDIMs} are simple cases of pseudo numerical methods. We change several classical numerical methods to corresponding pseudo numerical methods and find that the pseudo linear multi-step method is the best in most situations. According to our experiments, by directly using pre-trained models on Cifar10, {CelebA} and {LSUN}, {PNDMs} can generate higher quality synthetic images with only 50 steps compared with 1000-step {DDIMs} (20x speedup), significantly outperform {DDIMs} with 250 steps (by around 0.4 in {FID}) and have good generalization on different variance schedules. Our implementation is available at https://github.com/luping-liu/{PNDM}.},
	number = {{arXiv}:2202.09778},
	publisher = {{arXiv}},
	author = {Liu, Luping and Ren, Yi and Lin, Zhijie and Zhao, Zhou},
	urldate = {2022-09-27},
	date = {2022-02-20},
	eprinttype = {arxiv},
	eprint = {2202.09778 [cs, math, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning, Mathematics - Numerical Analysis},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/3WHYN88M/Liu et al. - 2022 - Pseudo Numerical Methods for Diffusion Models on M.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/I7C26YTQ/2202.html:text/html},
}

@misc{nichol_improved_2021,
	title = {Improved Denoising Diffusion Probabilistic Models},
	url = {http://arxiv.org/abs/2102.09672},
	doi = {10.48550/arXiv.2102.09672},
	abstract = {Denoising diffusion probabilistic models ({DDPM}) are a class of generative models which have recently been shown to produce excellent samples. We show that with a few simple modifications, {DDPMs} can also achieve competitive log-likelihoods while maintaining high sample quality. Additionally, we find that learning variances of the reverse diffusion process allows sampling with an order of magnitude fewer forward passes with a negligible difference in sample quality, which is important for the practical deployment of these models. We additionally use precision and recall to compare how well {DDPMs} and {GANs} cover the target distribution. Finally, we show that the sample quality and likelihood of these models scale smoothly with model capacity and training compute, making them easily scalable. We release our code at https://github.com/openai/improved-diffusion},
	number = {{arXiv}:2102.09672},
	publisher = {{arXiv}},
	author = {Nichol, Alex and Dhariwal, Prafulla},
	urldate = {2022-09-28},
	date = {2021-02-18},
	eprinttype = {arxiv},
	eprint = {2102.09672 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/Q8BFRFLC/Nichol and Dhariwal - 2021 - Improved Denoising Diffusion Probabilistic Models.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/WJ66NXIG/2102.html:text/html},
}

@misc{daras_your_2019,
	title = {Your Local {GAN}: Designing Two Dimensional Local Attention Mechanisms for Generative Models},
	url = {http://arxiv.org/abs/1911.12287},
	doi = {10.48550/arXiv.1911.12287},
	shorttitle = {Your Local {GAN}},
	abstract = {We introduce a new local sparse attention layer that preserves two-dimensional geometry and locality. We show that by just replacing the dense attention layer of {SAGAN} with our construction, we obtain very significant {FID}, Inception score and pure visual improvements. {FID} score is improved from \$18.65\$ to \$15.94\$ on {ImageNet}, keeping all other parameters the same. The sparse attention patterns that we propose for our new layer are designed using a novel information theoretic criterion that uses information flow graphs. We also present a novel way to invert Generative Adversarial Networks with attention. Our method extracts from the attention layer of the discriminator a saliency map, which we use to construct a new loss function for the inversion. This allows us to visualize the newly introduced attention heads and show that they indeed capture interesting aspects of two-dimensional geometry of real images.},
	number = {{arXiv}:1911.12287},
	publisher = {{arXiv}},
	author = {Daras, Giannis and Odena, Augustus and Zhang, Han and Dimakis, Alexandros G.},
	urldate = {2022-09-28},
	date = {2019-12-02},
	eprinttype = {arxiv},
	eprint = {1911.12287 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/HCYGJ2AJ/Daras et al. - 2019 - Your Local GAN Designing Two Dimensional Local At.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/ZX778KDI/1911.html:text/html},
}

@misc{johnson_perceptual_2016,
	title = {Perceptual Losses for Real-Time Style Transfer and Super-Resolution},
	url = {http://arxiv.org/abs/1603.08155},
	doi = {10.48550/arXiv.1603.08155},
	abstract = {We consider image transformation problems, where an input image is transformed into an output image. Recent methods for such problems typically train feed-forward convolutional neural networks using a {\textbackslash}emph\{per-pixel\} loss between the output and ground-truth images. Parallel work has shown that high-quality images can be generated by defining and optimizing {\textbackslash}emph\{perceptual\} loss functions based on high-level features extracted from pretrained networks. We combine the benefits of both approaches, and propose the use of perceptual loss functions for training feed-forward networks for image transformation tasks. We show results on image style transfer, where a feed-forward network is trained to solve the optimization problem proposed by Gatys et al in real-time. Compared to the optimization-based method, our network gives similar qualitative results but is three orders of magnitude faster. We also experiment with single-image super-resolution, where replacing a per-pixel loss with a perceptual loss gives visually pleasing results.},
	number = {{arXiv}:1603.08155},
	publisher = {{arXiv}},
	author = {Johnson, Justin and Alahi, Alexandre and Fei-Fei, Li},
	urldate = {2022-09-28},
	date = {2016-03-26},
	eprinttype = {arxiv},
	eprint = {1603.08155 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/6XF228Z3/Johnson et al. - 2016 - Perceptual Losses for Real-Time Style Transfer and.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/KP6SB24M/1603.html:text/html},
}

@misc{heitz_sliced_2021,
	title = {A Sliced Wasserstein Loss for Neural Texture Synthesis},
	url = {http://arxiv.org/abs/2006.07229},
	doi = {10.48550/arXiv.2006.07229},
	abstract = {We address the problem of computing a textural loss based on the statistics extracted from the feature activations of a convolutional neural network optimized for object recognition (e.g. {VGG}-19). The underlying mathematical problem is the measure of the distance between two distributions in feature space. The Gram-matrix loss is the ubiquitous approximation for this problem but it is subject to several shortcomings. Our goal is to promote the Sliced Wasserstein Distance as a replacement for it. It is theoretically proven,practical, simple to implement, and achieves results that are visually superior for texture synthesis by optimization or training generative neural networks.},
	number = {{arXiv}:2006.07229},
	publisher = {{arXiv}},
	author = {Heitz, Eric and Vanhoey, Kenneth and Chambon, Thomas and Belcour, Laurent},
	urldate = {2022-09-28},
	date = {2021-03-11},
	eprinttype = {arxiv},
	eprint = {2006.07229 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/4KSHIN8Q/Heitz et al. - 2021 - A Sliced Wasserstein Loss for Neural Texture Synth.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/GB2MF93J/2006.html:text/html},
}

@misc{ruiz_dreambooth_2022,
	title = {{DreamBooth}: Fine Tuning Text-to-Image Diffusion Models for Subject-Driven Generation},
	url = {http://arxiv.org/abs/2208.12242},
	doi = {10.48550/arXiv.2208.12242},
	shorttitle = {{DreamBooth}},
	abstract = {Large text-to-image models achieved a remarkable leap in the evolution of {AI}, enabling high-quality and diverse synthesis of images from a given text prompt. However, these models lack the ability to mimic the appearance of subjects in a given reference set and synthesize novel renditions of them in different contexts. In this work, we present a new approach for "personalization" of text-to-image diffusion models (specializing them to users' needs). Given as input just a few images of a subject, we fine-tune a pretrained text-to-image model (Imagen, although our method is not limited to a specific model) such that it learns to bind a unique identifier with that specific subject. Once the subject is embedded in the output domain of the model, the unique identifier can then be used to synthesize fully-novel photorealistic images of the subject contextualized in different scenes. By leveraging the semantic prior embedded in the model with a new autogenous class-specific prior preservation loss, our technique enables synthesizing the subject in diverse scenes, poses, views, and lighting conditions that do not appear in the reference images. We apply our technique to several previously-unassailable tasks, including subject recontextualization, text-guided view synthesis, appearance modification, and artistic rendering (all while preserving the subject's key features). Project page: https://dreambooth.github.io/},
	number = {{arXiv}:2208.12242},
	publisher = {{arXiv}},
	author = {Ruiz, Nataniel and Li, Yuanzhen and Jampani, Varun and Pritch, Yael and Rubinstein, Michael and Aberman, Kfir},
	urldate = {2022-09-28},
	date = {2022-08-25},
	eprinttype = {arxiv},
	eprint = {2208.12242 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Graphics},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/M3H2L4VN/Ruiz et al. - 2022 - DreamBooth Fine Tuning Text-to-Image Diffusion Mo.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/Z4A6Y3KH/2208.html:text/html},
}

@article{marron_style_nodate,
	title = {Style Transfer as Optimal Transport},
	abstract = {We propose a new loss function for performing style transfer, whereby a subject image is transformed to exhibit the visual style of a reference image. The style of an image is conceptualized as the distribution of a random vector in the space of convolution feature activations and the distance between two images’ styles is measured by the distance between their associated distributions. Specifically, we measure the first two sample moments of the convolution feature activations induced by subject and reference images and the L2-Wasserstein metric between Gaussians parameterized by the respective moments serves as our distance. The gradient of this distance with respect to the subject image provides an optimal transport plan, instructing how the subject image should be changed to exhibit the style of the reference image. This framework advances the technique of Gatys et al. [2015] by incorporating transportation theory.},
	author = {Marron, Vincent},
	urldate = {2022-09-29},
	file = {style-transfer-theory.pdf:/Users/seem/Zotero/storage/EZLATVJP/style-transfer-theory.pdf:application/pdf},
}

@misc{khrulkov_understanding_2022,
	title = {Understanding {DDPM} Latent Codes Through Optimal Transport},
	url = {http://arxiv.org/abs/2202.07477},
	doi = {10.48550/arXiv.2202.07477},
	abstract = {Diffusion models have recently outperformed alternative approaches to model the distribution of natural images, such as {GANs}. Such diffusion models allow for deterministic sampling via the probability flow {ODE}, giving rise to a latent space and an encoder map. While having important practical applications, such as estimation of the likelihood, the theoretical properties of this map are not yet fully understood. In the present work, we partially address this question for the popular case of the {VP} {SDE} ({DDPM}) approach. We show that, perhaps surprisingly, the {DDPM} encoder map coincides with the optimal transport map for common distributions; we support this claim theoretically and by extensive numerical experiments.},
	number = {{arXiv}:2202.07477},
	publisher = {{arXiv}},
	author = {Khrulkov, Valentin and Oseledets, Ivan},
	urldate = {2022-09-29},
	date = {2022-02-14},
	eprinttype = {arxiv},
	eprint = {2202.07477 [cs, math, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence, Mathematics - Numerical Analysis, Mathematics - Analysis of {PDEs}},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/CCTW5Z2L/Khrulkov and Oseledets - 2022 - Understanding DDPM Latent Codes Through Optimal Tr.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/YESUAVTA/2202.html:text/html},
}

@misc{jolicoeur-martineau_relativistic_2018,
	title = {The relativistic discriminator: a key element missing from standard {GAN}},
	url = {http://arxiv.org/abs/1807.00734},
	doi = {10.48550/arXiv.1807.00734},
	shorttitle = {The relativistic discriminator},
	abstract = {In standard generative adversarial network ({SGAN}), the discriminator estimates the probability that the input data is real. The generator is trained to increase the probability that fake data is real. We argue that it should also simultaneously decrease the probability that real data is real because 1) this would account for a priori knowledge that half of the data in the mini-batch is fake, 2) this would be observed with divergence minimization, and 3) in optimal settings, {SGAN} would be equivalent to integral probability metric ({IPM}) {GANs}. We show that this property can be induced by using a relativistic discriminator which estimate the probability that the given real data is more realistic than a randomly sampled fake data. We also present a variant in which the discriminator estimate the probability that the given real data is more realistic than fake data, on average. We generalize both approaches to non-standard {GAN} loss functions and we refer to them respectively as Relativistic {GANs} ({RGANs}) and Relativistic average {GANs} ({RaGANs}). We show that {IPM}-based {GANs} are a subset of {RGANs} which use the identity function. Empirically, we observe that 1) {RGANs} and {RaGANs} are significantly more stable and generate higher quality data samples than their non-relativistic counterparts, 2) Standard {RaGAN} with gradient penalty generate data of better quality than {WGAN}-{GP} while only requiring a single discriminator update per generator update (reducing the time taken for reaching the state-of-the-art by 400\%), and 3) {RaGANs} are able to generate plausible high resolutions images (256x256) from a very small sample (N=2011), while {GAN} and {LSGAN} cannot; these images are of significantly better quality than the ones generated by {WGAN}-{GP} and {SGAN} with spectral normalization.},
	number = {{arXiv}:1807.00734},
	publisher = {{arXiv}},
	author = {Jolicoeur-Martineau, Alexia},
	urldate = {2022-09-29},
	date = {2018-09-10},
	eprinttype = {arxiv},
	eprint = {1807.00734 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence, Computer Science - Cryptography and Security},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/3BRZ6DG5/Jolicoeur-Martineau - 2018 - The relativistic discriminator a key element miss.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/48C66CA3/1807.html:text/html},
}

@misc{wang_towards_2021,
	title = {Towards Real-World Blind Face Restoration with Generative Facial Prior},
	url = {http://arxiv.org/abs/2101.04061},
	doi = {10.48550/arXiv.2101.04061},
	abstract = {Blind face restoration usually relies on facial priors, such as facial geometry prior or reference prior, to restore realistic and faithful details. However, very low-quality inputs cannot offer accurate geometric prior while high-quality references are inaccessible, limiting the applicability in real-world scenarios. In this work, we propose {GFP}-{GAN} that leverages rich and diverse priors encapsulated in a pretrained face {GAN} for blind face restoration. This Generative Facial Prior ({GFP}) is incorporated into the face restoration process via novel channel-split spatial feature transform layers, which allow our method to achieve a good balance of realness and fidelity. Thanks to the powerful generative facial prior and delicate designs, our {GFP}-{GAN} could jointly restore facial details and enhance colors with just a single forward pass, while {GAN} inversion methods require expensive image-specific optimization at inference. Extensive experiments show that our method achieves superior performance to prior art on both synthetic and real-world datasets.},
	number = {{arXiv}:2101.04061},
	publisher = {{arXiv}},
	author = {Wang, Xintao and Li, Yu and Zhang, Honglun and Shan, Ying},
	urldate = {2022-09-29},
	date = {2021-06-10},
	eprinttype = {arxiv},
	eprint = {2101.04061 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/TWYCJZZ4/Wang et al. - 2021 - Towards Real-World Blind Face Restoration with Gen.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/QSUG3VKD/2101.html:text/html},
}

@inproceedings{anonymous_training-free_2022,
	title = {Training-Free Structured Diffusion Guidance for Compositional Text-to-Image Synthesis},
	url = {https://openreview.net/forum?id=PUIqjT4rzq7},
	abstract = {We propose a training-free approach to incorporate language structured for compositional text-to-image synthesis},
	eventtitle = {Submitted to The Eleventh International Conference on Learning Representations},
	author = {Anonymous},
	urldate = {2022-09-30},
	date = {2022-09-29},
	langid = {english},
	file = {Full Text PDF:/Users/seem/Zotero/storage/XW3NW7BZ/Anonymous - 2022 - Training-Free Structured Diffusion Guidance for Co.pdf:application/pdf;Snapshot:/Users/seem/Zotero/storage/5G4PFJP7/forum.html:text/html},
}

@misc{poole_dreamfusion_2022,
	title = {{DreamFusion}: Text-to-3D using 2D Diffusion},
	url = {http://arxiv.org/abs/2209.14988},
	doi = {10.48550/arXiv.2209.14988},
	shorttitle = {{DreamFusion}},
	abstract = {Recent breakthroughs in text-to-image synthesis have been driven by diffusion models trained on billions of image-text pairs. Adapting this approach to 3D synthesis would require large-scale datasets of labeled 3D data and efficient architectures for denoising 3D data, neither of which currently exist. In this work, we circumvent these limitations by using a pretrained 2D text-to-image diffusion model to perform text-to-3D synthesis. We introduce a loss based on probability density distillation that enables the use of a 2D diffusion model as a prior for optimization of a parametric image generator. Using this loss in a {DeepDream}-like procedure, we optimize a randomly-initialized 3D model (a Neural Radiance Field, or {NeRF}) via gradient descent such that its 2D renderings from random angles achieve a low loss. The resulting 3D model of the given text can be viewed from any angle, relit by arbitrary illumination, or composited into any 3D environment. Our approach requires no 3D training data and no modifications to the image diffusion model, demonstrating the effectiveness of pretrained image diffusion models as priors.},
	number = {{arXiv}:2209.14988},
	publisher = {{arXiv}},
	author = {Poole, Ben and Jain, Ajay and Barron, Jonathan T. and Mildenhall, Ben},
	urldate = {2022-09-30},
	date = {2022-09-29},
	eprinttype = {arxiv},
	eprint = {2209.14988 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/KV4AFZGI/Poole et al. - 2022 - DreamFusion Text-to-3D using 2D Diffusion.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/3GS2UE7C/2209.html:text/html},
}

@misc{wang_real-esrgan_2021,
	title = {Real-{ESRGAN}: Training Real-World Blind Super-Resolution with Pure Synthetic Data},
	url = {http://arxiv.org/abs/2107.10833},
	doi = {10.48550/arXiv.2107.10833},
	shorttitle = {Real-{ESRGAN}},
	abstract = {Though many attempts have been made in blind super-resolution to restore low-resolution images with unknown and complex degradations, they are still far from addressing general real-world degraded images. In this work, we extend the powerful {ESRGAN} to a practical restoration application (namely, Real-{ESRGAN}), which is trained with pure synthetic data. Specifically, a high-order degradation modeling process is introduced to better simulate complex real-world degradations. We also consider the common ringing and overshoot artifacts in the synthesis process. In addition, we employ a U-Net discriminator with spectral normalization to increase discriminator capability and stabilize the training dynamics. Extensive comparisons have shown its superior visual performance than prior works on various real datasets. We also provide efficient implementations to synthesize training pairs on the fly.},
	number = {{arXiv}:2107.10833},
	publisher = {{arXiv}},
	author = {Wang, Xintao and Xie, Liangbin and Dong, Chao and Shan, Ying},
	urldate = {2022-09-29},
	date = {2021-08-17},
	eprinttype = {arxiv},
	eprint = {2107.10833 [cs, eess]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Electrical Engineering and Systems Science - Image and Video Processing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/JHFALNSF/Wang et al. - 2021 - Real-ESRGAN Training Real-World Blind Super-Resol.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/79B548DK/2107.html:text/html},
}

@misc{zhang_designing_2021,
	title = {Designing a Practical Degradation Model for Deep Blind Image Super-Resolution},
	url = {http://arxiv.org/abs/2103.14006},
	doi = {10.48550/arXiv.2103.14006},
	abstract = {It is widely acknowledged that single image super-resolution ({SISR}) methods would not perform well if the assumed degradation model deviates from those in real images. Although several degradation models take additional factors into consideration, such as blur, they are still not effective enough to cover the diverse degradations of real images. To address this issue, this paper proposes to design a more complex but practical degradation model that consists of randomly shuffled blur, downsampling and noise degradations. Specifically, the blur is approximated by two convolutions with isotropic and anisotropic Gaussian kernels; the downsampling is randomly chosen from nearest, bilinear and bicubic interpolations; the noise is synthesized by adding Gaussian noise with different noise levels, adopting {JPEG} compression with different quality factors, and generating processed camera sensor noise via reverse-forward camera image signal processing ({ISP}) pipeline model and {RAW} image noise model. To verify the effectiveness of the new degradation model, we have trained a deep blind {ESRGAN} super-resolver and then applied it to super-resolve both synthetic and real images with diverse degradations. The experimental results demonstrate that the new degradation model can help to significantly improve the practicability of deep super-resolvers, thus providing a powerful alternative solution for real {SISR} applications.},
	number = {{arXiv}:2103.14006},
	publisher = {{arXiv}},
	author = {Zhang, Kai and Liang, Jingyun and Van Gool, Luc and Timofte, Radu},
	urldate = {2022-09-29},
	date = {2021-09-30},
	eprinttype = {arxiv},
	eprint = {2103.14006 [cs, eess]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Electrical Engineering and Systems Science - Image and Video Processing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/GZCK8YTK/Zhang et al. - 2021 - Designing a Practical Degradation Model for Deep B.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/W7HTJ9SV/2103.html:text/html},
}

@misc{park_contrastive_2020,
	title = {Contrastive Learning for Unpaired Image-to-Image Translation},
	url = {http://arxiv.org/abs/2007.15651},
	doi = {10.48550/arXiv.2007.15651},
	abstract = {In image-to-image translation, each patch in the output should reflect the content of the corresponding patch in the input, independent of domain. We propose a straightforward method for doing so -- maximizing mutual information between the two, using a framework based on contrastive learning. The method encourages two elements (corresponding patches) to map to a similar point in a learned feature space, relative to other elements (other patches) in the dataset, referred to as negatives. We explore several critical design choices for making contrastive learning effective in the image synthesis setting. Notably, we use a multilayer, patch-based approach, rather than operate on entire images. Furthermore, we draw negatives from within the input image itself, rather than from the rest of the dataset. We demonstrate that our framework enables one-sided translation in the unpaired image-to-image translation setting, while improving quality and reducing training time. In addition, our method can even be extended to the training setting where each "domain" is only a single image.},
	number = {{arXiv}:2007.15651},
	publisher = {{arXiv}},
	author = {Park, Taesung and Efros, Alexei A. and Zhang, Richard and Zhu, Jun-Yan},
	urldate = {2022-09-30},
	date = {2020-08-20},
	eprinttype = {arxiv},
	eprint = {2007.15651 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/P2XEQX5B/Park et al. - 2020 - Contrastive Learning for Unpaired Image-to-Image T.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/GAH2IKQK/2007.html:text/html},
}

@misc{spiegl_contrastive_2021,
	title = {Contrastive Unpaired Translation using Focal Loss for Patch Classification},
	url = {http://arxiv.org/abs/2109.12431},
	doi = {10.48550/arXiv.2109.12431},
	abstract = {Image-to-image translation models transfer images from input domain to output domain in an endeavor to retain the original content of the image. Contrastive Unpaired Translation is one of the existing methods for solving such problems. Significant advantage of this method, compared to competitors, is the ability to train and perform well in cases where both input and output domains are only a single image. Another key thing that differentiates this method from its predecessors is the usage of image patches rather than the whole images. It also turns out that sampling negatives (patches required to calculate the loss) from the same image achieves better results than a scenario where the negatives are sampled from other images in the dataset. This type of approach encourages mapping of corresponding patches to the same location in relation to other patches (negatives) while at the same time improves the output image quality and significantly decreases memory usage as well as the time required to train the model compared to {CycleGAN} method used as a baseline. Through a series of experiments we show that using focal loss in place of cross-entropy loss within the {PatchNCE} loss can improve on the model's performance and even surpass the current state-of-the-art model for image-to-image translation.},
	number = {{arXiv}:2109.12431},
	publisher = {{arXiv}},
	author = {Spiegl, Bernard},
	urldate = {2022-09-30},
	date = {2021-09-25},
	eprinttype = {arxiv},
	eprint = {2109.12431 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/T5TFSFFZ/Spiegl - 2021 - Contrastive Unpaired Translation using Focal Loss .pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/ACJ84QW2/2109.html:text/html},
}

@misc{sasaki_unit-ddpm_2021,
	title = {{UNIT}-{DDPM}: {UNpaired} Image Translation with Denoising Diffusion Probabilistic Models},
	url = {http://arxiv.org/abs/2104.05358},
	doi = {10.48550/arXiv.2104.05358},
	shorttitle = {{UNIT}-{DDPM}},
	abstract = {We propose a novel unpaired image-to-image translation method that uses denoising diffusion probabilistic models without requiring adversarial training. Our method, {UNpaired} Image Translation with Denoising Diffusion Probabilistic Models ({UNIT}-{DDPM}), trains a generative model to infer the joint distribution of images over both domains as a Markov chain by minimising a denoising score matching objective conditioned on the other domain. In particular, we update both domain translation models simultaneously, and we generate target domain images by a denoising Markov Chain Monte Carlo approach that is conditioned on the input source domain images, based on Langevin dynamics. Our approach provides stable model training for image-to-image translation and generates high-quality image outputs. This enables state-of-the-art Fr{\textbackslash}'echet Inception Distance ({FID}) performance on several public datasets, including both colour and multispectral imagery, significantly outperforming the contemporary adversarial image-to-image translation methods.},
	number = {{arXiv}:2104.05358},
	publisher = {{arXiv}},
	author = {Sasaki, Hiroshi and Willcocks, Chris G. and Breckon, Toby P.},
	urldate = {2022-09-30},
	date = {2021-04-12},
	eprinttype = {arxiv},
	eprint = {2104.05358 [cs, eess]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Electrical Engineering and Systems Science - Image and Video Processing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/G5FVBIP8/Sasaki et al. - 2021 - UNIT-DDPM UNpaired Image Translation with Denoisi.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/SGCLPQHD/2104.html:text/html},
}

@misc{meng_sdedit_2022,
	title = {{SDEdit}: Guided Image Synthesis and Editing with Stochastic Differential Equations},
	url = {http://arxiv.org/abs/2108.01073},
	doi = {10.48550/arXiv.2108.01073},
	shorttitle = {{SDEdit}},
	abstract = {Guided image synthesis enables everyday users to create and edit photo-realistic images with minimum effort. The key challenge is balancing faithfulness to the user input (e.g., hand-drawn colored strokes) and realism of the synthesized image. Existing {GAN}-based methods attempt to achieve such balance using either conditional {GANs} or {GAN} inversions, which are challenging and often require additional training data or loss functions for individual applications. To address these issues, we introduce a new image synthesis and editing method, Stochastic Differential Editing ({SDEdit}), based on a diffusion model generative prior, which synthesizes realistic images by iteratively denoising through a stochastic differential equation ({SDE}). Given an input image with user guide of any type, {SDEdit} first adds noise to the input, then subsequently denoises the resulting image through the {SDE} prior to increase its realism. {SDEdit} does not require task-specific training or inversions and can naturally achieve the balance between realism and faithfulness. {SDEdit} significantly outperforms state-of-the-art {GAN}-based methods by up to 98.09\% on realism and 91.72\% on overall satisfaction scores, according to a human perception study, on multiple tasks, including stroke-based image synthesis and editing as well as image compositing.},
	number = {{arXiv}:2108.01073},
	publisher = {{arXiv}},
	author = {Meng, Chenlin and He, Yutong and Song, Yang and Song, Jiaming and Wu, Jiajun and Zhu, Jun-Yan and Ermon, Stefano},
	urldate = {2022-09-30},
	date = {2022-01-04},
	eprinttype = {arxiv},
	eprint = {2108.01073 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/RHW4KXMI/Meng et al. - 2022 - SDEdit Guided Image Synthesis and Editing with St.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/Y28IXGDN/2108.html:text/html},
}

@misc{dhariwal_diffusion_2021,
	title = {Diffusion Models Beat {GANs} on Image Synthesis},
	url = {http://arxiv.org/abs/2105.05233},
	doi = {10.48550/arXiv.2105.05233},
	abstract = {We show that diffusion models can achieve image sample quality superior to the current state-of-the-art generative models. We achieve this on unconditional image synthesis by finding a better architecture through a series of ablations. For conditional image synthesis, we further improve sample quality with classifier guidance: a simple, compute-efficient method for trading off diversity for fidelity using gradients from a classifier. We achieve an {FID} of 2.97 on {ImageNet} 128\${\textbackslash}times\$128, 4.59 on {ImageNet} 256\${\textbackslash}times\$256, and 7.72 on {ImageNet} 512\${\textbackslash}times\$512, and we match {BigGAN}-deep even with as few as 25 forward passes per sample, all while maintaining better coverage of the distribution. Finally, we find that classifier guidance combines well with upsampling diffusion models, further improving {FID} to 3.94 on {ImageNet} 256\${\textbackslash}times\$256 and 3.85 on {ImageNet} 512\${\textbackslash}times\$512. We release our code at https://github.com/openai/guided-diffusion},
	number = {{arXiv}:2105.05233},
	publisher = {{arXiv}},
	author = {Dhariwal, Prafulla and Nichol, Alex},
	urldate = {2022-09-30},
	date = {2021-06-01},
	eprinttype = {arxiv},
	eprint = {2105.05233 [cs, stat]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/ILJ6Q3KF/Dhariwal and Nichol - 2021 - Diffusion Models Beat GANs on Image Synthesis.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/KPGGXGZS/2105.html:text/html},
}

@misc{zhang_fast_2022,
	title = {Fast Sampling of Diffusion Models with Exponential Integrator},
	url = {http://arxiv.org/abs/2204.13902},
	doi = {10.48550/arXiv.2204.13902},
	abstract = {The past few years have witnessed the great success of Diffusion models{\textasciitilde}({DMs}) in generating high-fidelity samples in generative modeling tasks. A major limitation of the {DM} is its notoriously slow sampling procedure which normally requires hundreds to thousands of time discretization steps of the learned diffusion process to reach the desired accuracy. Our goal is to develop a fast sampling method for {DMs} with a much less number of steps while retaining high sample quality. To this end, we systematically analyze the sampling procedure in {DMs} and identify key factors that affect the sample quality, among which the method of discretization is most crucial. By carefully examining the learned diffusion process, we propose Diffusion Exponential Integrator Sampler{\textasciitilde}({DEIS}). It is based on the Exponential Integrator designed for discretizing ordinary differential equations ({ODEs}) and leverages a semilinear structure of the learned diffusion process to reduce the discretization error. The proposed method can be applied to any {DMs} and can generate high-fidelity samples in as few as 10 steps. In our experiments, it takes about 3 minutes on one A6000 {GPU} to generate \$50k\$ images from {CIFAR}10. Moreover, by directly using pre-trained {DMs}, we achieve the state-of-art sampling performance when the number of score function evaluation{\textasciitilde}({NFE}) is limited, e.g., 4.17 {FID} with 10 {NFEs}, 3.37 {FID}, and 9.74 {IS} with only 15 {NFEs} on {CIFAR}10. Code is available at https://github.com/qsh-zh/deis},
	number = {{arXiv}:2204.13902},
	publisher = {{arXiv}},
	author = {Zhang, Qinsheng and Chen, Yongxin},
	urldate = {2022-09-30},
	date = {2022-06-18},
	eprinttype = {arxiv},
	eprint = {2204.13902 [cs]},
	keywords = {Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/ZK4IIA7Y/Zhang and Chen - 2022 - Fast Sampling of Diffusion Models with Exponential.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/BPLCTDZL/2204.html:text/html},
}

@misc{hertz_prompt--prompt_2022,
	title = {Prompt-to-Prompt Image Editing with Cross Attention Control},
	url = {http://arxiv.org/abs/2208.01626},
	doi = {10.48550/arXiv.2208.01626},
	abstract = {Recent large-scale text-driven synthesis models have attracted much attention thanks to their remarkable capabilities of generating highly diverse images that follow given text prompts. Such text-based synthesis methods are particularly appealing to humans who are used to verbally describe their intent. Therefore, it is only natural to extend the text-driven image synthesis to text-driven image editing. Editing is challenging for these generative models, since an innate property of an editing technique is to preserve most of the original image, while in the text-based models, even a small modification of the text prompt often leads to a completely different outcome. State-of-the-art methods mitigate this by requiring the users to provide a spatial mask to localize the edit, hence, ignoring the original structure and content within the masked region. In this paper, we pursue an intuitive prompt-to-prompt editing framework, where the edits are controlled by text only. To this end, we analyze a text-conditioned model in depth and observe that the cross-attention layers are the key to controlling the relation between the spatial layout of the image to each word in the prompt. With this observation, we present several applications which monitor the image synthesis by editing the textual prompt only. This includes localized editing by replacing a word, global editing by adding a specification, and even delicately controlling the extent to which a word is reflected in the image. We present our results over diverse images and prompts, demonstrating high-quality synthesis and fidelity to the edited prompts.},
	number = {{arXiv}:2208.01626},
	publisher = {{arXiv}},
	author = {Hertz, Amir and Mokady, Ron and Tenenbaum, Jay and Aberman, Kfir and Pritch, Yael and Cohen-Or, Daniel},
	urldate = {2022-09-30},
	date = {2022-08-02},
	eprinttype = {arxiv},
	eprint = {2208.01626 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Graphics, Computer Science - Computation and Language},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/XQYVUADN/Hertz et al. - 2022 - Prompt-to-Prompt Image Editing with Cross Attentio.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/M4JBEN9C/2208.html:text/html},
}

@misc{kingma_auto-encoding_2014,
	title = {Auto-Encoding Variational Bayes},
	url = {http://arxiv.org/abs/1312.6114},
	doi = {10.48550/arXiv.1312.6114},
	abstract = {How can we perform efficient inference and learning in directed probabilistic models, in the presence of continuous latent variables with intractable posterior distributions, and large datasets? We introduce a stochastic variational inference and learning algorithm that scales to large datasets and, under some mild differentiability conditions, even works in the intractable case. Our contributions is two-fold. First, we show that a reparameterization of the variational lower bound yields a lower bound estimator that can be straightforwardly optimized using standard stochastic gradient methods. Second, we show that for i.i.d. datasets with continuous latent variables per datapoint, posterior inference can be made especially efficient by fitting an approximate inference model (also called a recognition model) to the intractable posterior using the proposed lower bound estimator. Theoretical advantages are reflected in experimental results.},
	number = {{arXiv}:1312.6114},
	publisher = {{arXiv}},
	author = {Kingma, Diederik P. and Welling, Max},
	urldate = {2022-09-30},
	date = {2014-05-01},
	eprinttype = {arxiv},
	eprint = {1312.6114 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/PWFSB99M/Kingma and Welling - 2014 - Auto-Encoding Variational Bayes.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/I6BNNWHS/1312.html:text/html},
}

@misc{sohl-dickstein_deep_2015,
	title = {Deep Unsupervised Learning using Nonequilibrium Thermodynamics},
	url = {http://arxiv.org/abs/1503.03585},
	doi = {10.48550/arXiv.1503.03585},
	abstract = {A central problem in machine learning involves modeling complex data-sets using highly flexible families of probability distributions in which learning, sampling, inference, and evaluation are still analytically or computationally tractable. Here, we develop an approach that simultaneously achieves both flexibility and tractability. The essential idea, inspired by non-equilibrium statistical physics, is to systematically and slowly destroy structure in a data distribution through an iterative forward diffusion process. We then learn a reverse diffusion process that restores structure in data, yielding a highly flexible and tractable generative model of the data. This approach allows us to rapidly learn, sample from, and evaluate probabilities in deep generative models with thousands of layers or time steps, as well as to compute conditional and posterior probabilities under the learned model. We additionally release an open source reference implementation of the algorithm.},
	number = {{arXiv}:1503.03585},
	publisher = {{arXiv}},
	author = {Sohl-Dickstein, Jascha and Weiss, Eric A. and Maheswaranathan, Niru and Ganguli, Surya},
	urldate = {2022-10-01},
	date = {2015-11-18},
	eprinttype = {arxiv},
	eprint = {1503.03585 [cond-mat, q-bio, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Condensed Matter - Disordered Systems and Neural Networks, Quantitative Biology - Neurons and Cognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/2PEH6MBE/Sohl-Dickstein et al. - 2015 - Deep Unsupervised Learning using Nonequilibrium Th.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/X82B84N9/1503.html:text/html},
}

@misc{bostrom_byte_2020,
	title = {Byte Pair Encoding is Suboptimal for Language Model Pretraining},
	url = {http://arxiv.org/abs/2004.03720},
	doi = {10.48550/arXiv.2004.03720},
	abstract = {The success of pretrained transformer language models ({LMs}) in natural language processing has led to a wide range of pretraining setups. In particular, these models employ a variety of subword tokenization methods, most notably byte-pair encoding ({BPE}) (Sennrich et al., 2016; Gage, 1994), the {WordPiece} method (Schuster and Nakajima, 2012), and unigram language modeling (Kudo, 2018), to segment text. However, to the best of our knowledge, the literature does not contain a direct evaluation of the impact of tokenization on language model pretraining. We analyze differences between {BPE} and unigram {LM} tokenization, finding that the latter method recovers subword units that align more closely with morphology and avoids problems stemming from {BPE}'s greedy construction procedure. We then compare the fine-tuned task performance of identical transformer masked language models pretrained with these tokenizations. Across downstream tasks and two languages (English and Japanese), we find that the unigram {LM} tokenization method matches or outperforms {BPE}. We hope that developers of future pretrained {LMs} will consider adopting the unigram {LM} method over the more prevalent {BPE}.},
	number = {{arXiv}:2004.03720},
	publisher = {{arXiv}},
	author = {Bostrom, Kaj and Durrett, Greg},
	urldate = {2022-10-02},
	date = {2020-10-05},
	eprinttype = {arxiv},
	eprint = {2004.03720 [cs]},
	keywords = {Computer Science - Computation and Language, I.2.7},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/Q925NDRH/Bostrom and Durrett - 2020 - Byte Pair Encoding is Suboptimal for Language Mode.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/Q7N2Z8NR/2004.html:text/html},
}

@misc{liang_swinir_2021,
	title = {{SwinIR}: Image Restoration Using Swin Transformer},
	url = {http://arxiv.org/abs/2108.10257},
	doi = {10.48550/arXiv.2108.10257},
	shorttitle = {{SwinIR}},
	abstract = {Image restoration is a long-standing low-level vision problem that aims to restore high-quality images from low-quality images (e.g., downscaled, noisy and compressed images). While state-of-the-art image restoration methods are based on convolutional neural networks, few attempts have been made with Transformers which show impressive performance on high-level vision tasks. In this paper, we propose a strong baseline model {SwinIR} for image restoration based on the Swin Transformer. {SwinIR} consists of three parts: shallow feature extraction, deep feature extraction and high-quality image reconstruction. In particular, the deep feature extraction module is composed of several residual Swin Transformer blocks ({RSTB}), each of which has several Swin Transformer layers together with a residual connection. We conduct experiments on three representative tasks: image super-resolution (including classical, lightweight and real-world image super-resolution), image denoising (including grayscale and color image denoising) and {JPEG} compression artifact reduction. Experimental results demonstrate that {SwinIR} outperforms state-of-the-art methods on different tasks by \${\textbackslash}textbf\{up to 0.14\${\textbackslash}sim\$0.45dB\}\$, while the total number of parameters can be reduced by \${\textbackslash}textbf\{up to 67\%\}\$.},
	number = {{arXiv}:2108.10257},
	publisher = {{arXiv}},
	author = {Liang, Jingyun and Cao, Jiezhang and Sun, Guolei and Zhang, Kai and Van Gool, Luc and Timofte, Radu},
	urldate = {2022-10-01},
	date = {2021-08-23},
	eprinttype = {arxiv},
	eprint = {2108.10257 [cs, eess]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Electrical Engineering and Systems Science - Image and Video Processing},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/6C8LU6TL/Liang et al. - 2021 - SwinIR Image Restoration Using Swin Transformer.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/GXC9BD3V/2108.html:text/html},
}

@misc{song_generative_2020,
	title = {Generative Modeling by Estimating Gradients of the Data Distribution},
	url = {http://arxiv.org/abs/1907.05600},
	doi = {10.48550/arXiv.1907.05600},
	abstract = {We introduce a new generative model where samples are produced via Langevin dynamics using gradients of the data distribution estimated with score matching. Because gradients can be ill-defined and hard to estimate when the data resides on low-dimensional manifolds, we perturb the data with different levels of Gaussian noise, and jointly estimate the corresponding scores, i.e., the vector fields of gradients of the perturbed data distribution for all noise levels. For sampling, we propose an annealed Langevin dynamics where we use gradients corresponding to gradually decreasing noise levels as the sampling process gets closer to the data manifold. Our framework allows flexible model architectures, requires no sampling during training or the use of adversarial methods, and provides a learning objective that can be used for principled model comparisons. Our models produce samples comparable to {GANs} on {MNIST}, {CelebA} and {CIFAR}-10 datasets, achieving a new state-of-the-art inception score of 8.87 on {CIFAR}-10. Additionally, we demonstrate that our models learn effective representations via image inpainting experiments.},
	number = {{arXiv}:1907.05600},
	publisher = {{arXiv}},
	author = {Song, Yang and Ermon, Stefano},
	urldate = {2022-10-03},
	date = {2020-10-10},
	eprinttype = {arxiv},
	eprint = {1907.05600 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/CLYLSX8N/Song and Ermon - 2020 - Generative Modeling by Estimating Gradients of the.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/7PM6ZBDL/1907.html:text/html;Full Text:/Users/seem/Zotero/storage/7A7A8JVV/Song and Ermon - 2020 - Generative Modeling by Estimating Gradients of the.pdf:application/pdf},
}

@misc{ding_cogview2_2022,
	title = {{CogView}2: Faster and Better Text-to-Image Generation via Hierarchical Transformers},
	url = {http://arxiv.org/abs/2204.14217},
	doi = {10.48550/arXiv.2204.14217},
	shorttitle = {{CogView}2},
	abstract = {The development of the transformer-based text-to-image models are impeded by its slow generation and complexity for high-resolution images. In this work, we put forward a solution based on hierarchical transformers and local parallel auto-regressive generation. We pretrain a 6B-parameter transformer with a simple and flexible self-supervised task, Cross-modal general language model ({CogLM}), and finetune it for fast super-resolution. The new text-to-image system, {CogView}2, shows very competitive generation compared to concurrent state-of-the-art {DALL}-E-2, and naturally supports interactive text-guided editing on images.},
	number = {{arXiv}:2204.14217},
	publisher = {{arXiv}},
	author = {Ding, Ming and Zheng, Wendi and Hong, Wenyi and Tang, Jie},
	urldate = {2022-10-04},
	date = {2022-05-27},
	eprinttype = {arxiv},
	eprint = {2204.14217 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/H3LPJJRQ/Ding et al. - 2022 - CogView2 Faster and Better Text-to-Image Generati.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/EZFXFDRG/2204.html:text/html},
}

@misc{parmar_aliased_2022,
	title = {On Aliased Resizing and Surprising Subtleties in {GAN} Evaluation},
	url = {http://arxiv.org/abs/2104.11222},
	doi = {10.48550/arXiv.2104.11222},
	abstract = {Metrics for evaluating generative models aim to measure the discrepancy between real and generated images. The often-used Frechet Inception Distance ({FID}) metric, for example, extracts "high-level" features using a deep network from the two sets. However, we find that the differences in "low-level" preprocessing, specifically image resizing and compression, can induce large variations and have unforeseen consequences. For instance, when resizing an image, e.g., with a bilinear or bicubic kernel, signal processing principles mandate adjusting prefilter width depending on the downsampling factor, to antialias to the appropriate bandwidth. However, commonly-used implementations use a fixed-width prefilter, resulting in aliasing artifacts. Such aliasing leads to corruptions in the feature extraction downstream. Next, lossy compression, such as {JPEG}, is commonly used to reduce the file size of an image. Although designed to minimally degrade the perceptual quality of an image, the operation also produces variations downstream. Furthermore, we show that if compression is used on real training images, {FID} can actually improve if the generated images are also subsequently compressed. This paper shows that choices in low-level image processing have been an underappreciated aspect of generative modeling. We identify and characterize variations in generative modeling development pipelines, provide recommendations based on signal processing principles, and release a reference implementation to facilitate future comparisons.},
	number = {{arXiv}:2104.11222},
	publisher = {{arXiv}},
	author = {Parmar, Gaurav and Zhang, Richard and Zhu, Jun-Yan},
	urldate = {2022-10-04},
	date = {2022-01-20},
	eprinttype = {arxiv},
	eprint = {2104.11222 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Graphics},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/P2FH26Q2/Parmar et al. - 2022 - On Aliased Resizing and Surprising Subtleties in G.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/NSMBIY4X/2104.html:text/html},
}

@misc{beyer_better_2022,
	title = {Better plain {ViT} baselines for {ImageNet}-1k},
	url = {http://arxiv.org/abs/2205.01580},
	abstract = {It is commonly accepted that the Vision Transformer model requires sophisticated regularization techniques to excel at {ImageNet}-1k scale data. Surprisingly, we find this is not the case and standard data augmentation is sufficient. This note presents a few minor modifications to the original Vision Transformer ({ViT}) vanilla training setting that dramatically improve the performance of plain {ViT} models. Notably, 90 epochs of training surpass 76\% top-1 accuracy in under seven hours on a {TPUv}3-8, similar to the classic {ResNet}50 baseline, and 300 epochs of training reach 80\% in less than one day.},
	number = {{arXiv}:2205.01580},
	publisher = {{arXiv}},
	author = {Beyer, Lucas and Zhai, Xiaohua and Kolesnikov, Alexander},
	urldate = {2022-10-04},
	date = {2022-05-03},
	eprinttype = {arxiv},
	eprint = {2205.01580 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/DQ3NEGZC/Beyer et al. - 2022 - Better plain ViT baselines for ImageNet-1k.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/QSJTFQ5H/2205.html:text/html},
}

@misc{saharia_palette_2022,
	title = {Palette: Image-to-Image Diffusion Models},
	url = {http://arxiv.org/abs/2111.05826},
	doi = {10.48550/arXiv.2111.05826},
	shorttitle = {Palette},
	abstract = {This paper develops a unified framework for image-to-image translation based on conditional diffusion models and evaluates this framework on four challenging image-to-image translation tasks, namely colorization, inpainting, uncropping, and {JPEG} restoration. Our simple implementation of image-to-image diffusion models outperforms strong {GAN} and regression baselines on all tasks, without task-specific hyper-parameter tuning, architecture customization, or any auxiliary loss or sophisticated new techniques needed. We uncover the impact of an L2 vs. L1 loss in the denoising diffusion objective on sample diversity, and demonstrate the importance of self-attention in the neural architecture through empirical studies. Importantly, we advocate a unified evaluation protocol based on {ImageNet}, with human evaluation and sample quality scores ({FID}, Inception Score, Classification Accuracy of a pre-trained {ResNet}-50, and Perceptual Distance against original images). We expect this standardized evaluation protocol to play a role in advancing image-to-image translation research. Finally, we show that a generalist, multi-task diffusion model performs as well or better than task-specific specialist counterparts. Check out https://diffusion-palette.github.io for an overview of the results.},
	number = {{arXiv}:2111.05826},
	publisher = {{arXiv}},
	author = {Saharia, Chitwan and Chan, William and Chang, Huiwen and Lee, Chris A. and Ho, Jonathan and Salimans, Tim and Fleet, David J. and Norouzi, Mohammad},
	urldate = {2022-10-04},
	date = {2022-05-03},
	eprinttype = {arxiv},
	eprint = {2111.05826 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/TSZDVXDN/Saharia et al. - 2022 - Palette Image-to-Image Diffusion Models.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/2UKD9SAL/2111.html:text/html},
}

@misc{blattmann_retrieval-augmented_2022,
	title = {Retrieval-Augmented Diffusion Models},
	url = {http://arxiv.org/abs/2204.11824},
	abstract = {Generative image synthesis with diffusion models has recently achieved excellent visual quality in several tasks such as text-based or class-conditional image synthesis. Much of this success is due to a dramatic increase in the computational capacity invested in training these models. This work presents an alternative approach: inspired by its successful application in natural language processing, we propose to complement the diffusion model with a retrieval-based approach and to introduce an explicit memory in the form of an external database. During training, our diffusion model is trained with similar visual features retrieved via {CLIP} and from the neighborhood of each training instance. By leveraging {CLIP}'s joint image-text embedding space, our model achieves highly competitive performance on tasks for which it has not been explicitly trained, such as class-conditional or text-image synthesis, and can be conditioned on both text and image embeddings. Moreover, we can apply our approach to unconditional generation, where it achieves state-of-the-art performance. Our approach incurs low computational and memory overheads and is easy to implement. We discuss its relationship to concurrent work and will publish code and pretrained models soon.},
	number = {{arXiv}:2204.11824},
	publisher = {{arXiv}},
	author = {Blattmann, Andreas and Rombach, Robin and Oktay, Kaan and Ommer, Björn},
	urldate = {2022-10-04},
	date = {2022-04-26},
	eprinttype = {arxiv},
	eprint = {2204.11824 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition},
}

@misc{norelli_asif_2022,
	title = {{ASIF}: Coupled Data Turns Unimodal Models to Multimodal Without Training},
	url = {http://arxiv.org/abs/2210.01738},
	doi = {10.48550/arXiv.2210.01738},
	shorttitle = {{ASIF}},
	abstract = {Aligning the visual and language spaces requires to train deep neural networks from scratch on giant multimodal datasets; {CLIP} trains both an image and a text encoder, while {LiT} manages to train just the latter by taking advantage of a pretrained vision network. In this paper, we show that sparse relative representations are sufficient to align text and images without training any network. Our method relies on readily available single-domain encoders (trained with or without supervision) and a modest (in comparison) number of image-text pairs. {ASIF} redefines what constitutes a multimodal model by explicitly disentangling memory from processing: here the model is defined by the embedded pairs of all the entries in the multimodal dataset, in addition to the parameters of the two encoders. Experiments on standard zero-shot visual benchmarks demonstrate the typical transfer ability of image-text models. Overall, our method represents a simple yet surprisingly strong baseline for foundation multimodal models, raising important questions on their data efficiency and on the role of retrieval in machine learning.},
	number = {{arXiv}:2210.01738},
	publisher = {{arXiv}},
	author = {Norelli, Antonio and Fumero, Marco and Maiorca, Valentino and Moschella, Luca and Rodolà, Emanuele and Locatello, Francesco},
	urldate = {2022-10-05},
	date = {2022-10-04},
	eprinttype = {arxiv},
	eprint = {2210.01738 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Computer Science - Artificial Intelligence},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/N5VMDN5P/Norelli et al. - 2022 - ASIF Coupled Data Turns Unimodal Models to Multim.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/SVBGEDKU/2210.html:text/html},
}

@misc{harrison_closer_2022,
	title = {A Closer Look at Learned Optimization: Stability, Robustness, and Inductive Biases},
	url = {http://arxiv.org/abs/2209.11208},
	doi = {10.48550/arXiv.2209.11208},
	shorttitle = {A Closer Look at Learned Optimization},
	abstract = {Learned optimizers -- neural networks that are trained to act as optimizers -- have the potential to dramatically accelerate training of machine learning models. However, even when meta-trained across thousands of tasks at huge computational expense, blackbox learned optimizers often struggle with stability and generalization when applied to tasks unlike those in their meta-training set. In this paper, we use tools from dynamical systems to investigate the inductive biases and stability properties of optimization algorithms, and apply the resulting insights to designing inductive biases for blackbox optimizers. Our investigation begins with a noisy quadratic model, where we characterize conditions in which optimization is stable, in terms of eigenvalues of the training dynamics. We then introduce simple modifications to a learned optimizer's architecture and meta-training procedure which lead to improved stability, and improve the optimizer's inductive bias. We apply the resulting learned optimizer to a variety of neural network training tasks, where it outperforms the current state of the art learned optimizer -- at matched optimizer computational overhead -- with regard to optimization performance and meta-training speed, and is capable of generalization to tasks far different from those it was meta-trained on.},
	number = {{arXiv}:2209.11208},
	publisher = {{arXiv}},
	author = {Harrison, James and Metz, Luke and Sohl-Dickstein, Jascha},
	urldate = {2022-10-07},
	date = {2022-09-22},
	eprinttype = {arxiv},
	eprint = {2209.11208 [cs, math, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Mathematics - Optimization and Control},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/7RIFVH2P/Harrison et al. - 2022 - A Closer Look at Learned Optimization Stability, .pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/Z7HZXQLA/2209.html:text/html},
}

@misc{merullo_linearly_2022,
	title = {Linearly Mapping from Image to Text Space},
	url = {http://arxiv.org/abs/2209.15162},
	doi = {10.48550/arXiv.2209.15162},
	abstract = {The extent to which text-only language models ({LMs}) learn to represent the physical, non-linguistic world is an open question. Prior work has shown that pretrained {LMs} can be taught to ``understand'' visual inputs when the models' parameters are updated on image captioning tasks. We test a stronger hypothesis: that the conceptual representations learned by text-only models are functionally equivalent (up to a linear transformation) to those learned by models trained on vision tasks. Specifically, we show that the image representations from vision models can be transferred as continuous prompts to frozen {LMs} by training only a single linear projection. Using these to prompt the {LM} achieves competitive performance on captioning and visual question answering tasks compared to models that tune both the image encoder and text decoder (such as the {MAGMA} model). We compare three image encoders with increasing amounts of linguistic supervision seen during pretraining: {BEIT} (no linguistic information), {NF}-{ResNET} (lexical category information), and {CLIP} (full natural language descriptions). We find that all three encoders perform equally well at transferring visual property information to the language model (e.g., whether an animal is large or small), but that image encoders pretrained with linguistic supervision more saliently encode category information (e.g., distinguishing hippo vs.{\textbackslash} elephant) and thus perform significantly better on benchmark language-and-vision tasks. Our results indicate that {LMs} encode conceptual information structurally similarly to vision-based models, even those that are solely trained on images.},
	number = {{arXiv}:2209.15162},
	publisher = {{arXiv}},
	author = {Merullo, Jack and Castricato, Louis and Eickhoff, Carsten and Pavlick, Ellie},
	urldate = {2022-10-08},
	date = {2022-09-29},
	eprinttype = {arxiv},
	eprint = {2209.15162 [cs]},
	keywords = {Computer Science - Machine Learning, Computer Science - Computation and Language},
	file = {arXiv Fulltext PDF:/Users/seem/Zotero/storage/YQKAFHA5/Merullo et al. - 2022 - Linearly Mapping from Image to Text Space.pdf:application/pdf;arXiv.org Snapshot:/Users/seem/Zotero/storage/6ARJTNES/2209.html:text/html},
}