@inproceedings{8665231b163a4dcfa3dbdc7fe2a4d913,
title = "An information-theoretic analysis of the impact of task similarity on meta-learning",
abstract = "Meta-learning aims at optimizing the hyperparameters of a model class or training algorithm from the observation of data from a number of related tasks. Following the setting of Baxter [1], the tasks are assumed to belong to the same task environment, which is defined by a distribution over the space of tasks and by per-task data distributions. The statistical properties of the task environment thus dictate the similarity of the tasks. The goal of the meta-learner is to ensure that the hyperparameters obtain a small loss when applied for training of a new task sampled from the task environment. The difference between the resulting average loss, known as meta-population loss, and the corresponding empirical loss measured on the available data from related tasks, known as meta-generalization gap, is a measure of the generalization capability of the meta-learner. In this paper, we present novel information-theoretic bounds on the average absolute value of the meta-generalization gap. Unlike prior work [2], our bounds explicitly capture the impact of task relatedness, the number of tasks, and the number of data samples per task on the meta-generalization gap. Task similarity is gauged via the Kullback-Leibler (KL) and Jensen-Shannon (JS) divergences. We illustrate the proposed bounds on the example of ridge regression with meta-learned bias.",
keywords = "Training, Loss measurement, Data models, Task analysis, Information theory",
author = "Sharu Jose and Osvaldo Simeone",
year = "2021",
month = sep,
day = "1",
doi = "10.1109/ISIT45174.2021.9517767",
language = "English",
isbn = "9781538682104 (PoD)",
series = "IEEE International Symposium on Information Theory proceedings",
publisher = "IEEE",
pages = "1534--1539",
booktitle = "2021 IEEE International Symposium on Information Theory (ISIT)",
note = "2021 IEEE International Symposium on Information Theory, ISIT 2021 ; Conference date: 12-07-2021 Through 20-07-2021",
}