From 858fc7660341e872913a28002af721d6eca742f9 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Sun, 15 Nov 2026 23:35:50 +0200 Subject: [PATCH 01/40] Add alt text to the figures of the vignettes --- vignettes/latent.Rmd | 14 +++++++------- vignettes/projpred.Rmd | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 0bcdfac19..573ed784e 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -201,7 +201,7 @@ In order to decide for a submodel size, we first inspect the `plot()` results. In contrast to the main vignette where we used the mean log predictive density (MLPD) as predictive performance statistic for a `gaussian()` family reference model (and `gaussian()` submodels), we have a discrete family (`poisson()`) here, so it makes sense to exponentiate the MLPD to obtain the geometric mean predictive density (GMPD; in case of a discrete response, the predictive density values are actually predictive *probabilities* and hence the GMPD is bounded by 0 and 1). As in the main vignette, we plot with `deltas = TRUE` (in case of the GMPD, this means that the *ratio* of the submodel GMPD vs. the reference model GMPD is shown). Via global option `projpred.plot_vsel_size_position`, we set argument `size_position` of `plot.vsel()` to `"secondary_x"` to make the submodel sizes readable in all of the plots in this vignette. -```{r plot_vsel_lat} +```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to the GMPD of the reference model against the submodel size, with uncertainty intervals, for the latent projection of the Poisson model. A dashed horizontal line at 1 marks the reference model."} options(projpred.plot_vsel_size_position = "secondary_x") ( gg_lat <- plot(vs_lat, stats = "gmpd", deltas = TRUE) ) ``` @@ -258,7 +258,7 @@ time_trad <- system.time(vs_trad <- varsel( options(warn_instable_orig) rm(warn_instable_orig) ``` -```{r post_vs_trad} +```{r post_vs_trad, fig.alt="The same plot for the traditional projection of the Poisson model. Several noise terms enter the predictor ranking before relevant ones, and the submodels reach the GMPD of the reference model only at submodel size 28."} print(time_trad) ( gg_trad <- plot(vs_trad, stats = "gmpd", deltas = TRUE) ) ``` @@ -401,7 +401,7 @@ Again, the message telling that `$dis` consists of only `NA`s will not The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin} +```{r plot_vsel_nebin, fig.alt="Plot of the GMPD ratio of each submodel to the reference model against the submodel size, with uncertainty intervals, for the latent projection of the negative binomial model. A dashed horizontal line at 1 marks the reference model."} ( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) ``` @@ -587,13 +587,13 @@ In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values bec We will also ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. Plot the results: -```{r weibull_plot_cvvs} +```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size, with uncertainty intervals, for the Weibull model with right-censored observations. A dashed horizontal line at zero marks the reference model."} plot(cvvs_weib, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. Project onto the submodel consisting of the first `n_pred_truth` predictors and perform a "posterior-projection predictive check" (PPPC): -```{r weibull_pppc} +```{r weibull_pppc, fig.alt="Kaplan-Meier overlay plot for the Weibull submodel: the Kaplan-Meier estimate of the survival function of the observed times as a dark line, and the Kaplan-Meier estimates of the predictive draws from proj_predict() as light lines. The light lines are close to the dark line."} predictors_final_weib <- head(ranking(cvvs_weib)[["fulldata"]], n_pred_truth) prj_weib <- project(refm_weib, predictor_terms = predictors_final_weib) prj_predict_weib <- proj_predict(prj_weib) @@ -730,13 +730,13 @@ In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values bec We will also ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. Plot the results: -```{r lognormal_plot_cvvs} +```{r lognormal_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size, with uncertainty intervals, for the log-normal model with right-censored observations. A dashed horizontal line at zero marks the reference model."} plot(cvvs_lnorm, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. Project onto the submodel consisting of the first `n_pred_truth` predictors and perform a "posterior-projection predictive check" (PPPC): -```{r lognormal_pppc} +```{r lognormal_pppc, fig.alt="Kaplan-Meier overlay plot for the log-normal submodel: the Kaplan-Meier estimate of the survival function of the observed times as a dark line, and the Kaplan-Meier estimates of the predictive draws from proj_predict() as light lines. The light lines are close to the dark line."} predictors_final_lnorm <- head(ranking(cvvs_lnorm)[["fulldata"]], n_pred_truth) prj_lnorm <- project(refm_lnorm, predictor_terms = predictors_final_lnorm) prj_predict_lnorm <- proj_predict(prj_lnorm) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 04830bc6f..5fe5de08b 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -152,7 +152,7 @@ cvvs_fast <- cv_varsel( To find a suitable value for `nterms_max` in subsequent `cv_varsel()` runs, we take a look at a plot of at least one predictive performance statistic in dependence of the submodel size. Here, we choose the mean log predictive density (MLPD; see the documentation for argument `stats` of `summary.vsel()` for details) as the only performance statistic. Since we will be using the following plot only to determine `nterms_max` for subsequent `cv_varsel()` runs, we can omit the predictor ranking from the plot by setting `ranking_nterms_max` to `NA` (which requires `size_position = "primary_x_bottom"`, which we set here via its global option, just like argument `text_angle`): -```{r plot_vsel_fast} +```{r plot_vsel_fast, fig.alt="Line plot of the mean log predictive density (MLPD) of the submodels against the submodel size from 0 to 20, from the preliminary L1 search with refit_prj = FALSE. A dashed horizontal line shows the MLPD of the reference model. The submodel MLPD increases with the size and levels off from size 8 on."} options(projpred.plot_vsel_size_position = "primary_x_bottom") options(projpred.plot_vsel_text_angle = 0) plot(cvvs_fast, stats = "mlpd", ranking_nterms_max = NA) @@ -176,7 +176,7 @@ cvvs_fast_refit <- cv_varsel( Here, we ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. With the `refit_prj = TRUE` results, the predictive performance plot from above now looks as follows: -```{r plot_vsel_fast_refit} +```{r plot_vsel_fast_refit, fig.alt="The same plot as the previous one, from the preliminary run with refit_prj = TRUE. The submodel MLPD changes little after size 8."} plot(cvvs_fast_refit, stats = "mlpd", ranking_nterms_max = NA) ``` This refined plot confirms that the submodel MLPD does not change much after submodel size 8, so in our final `cv_varsel()` run, we set `nterms_max` to a value slightly higher than 8 (here: 9) to ensure that we see the MLPD leveling off. @@ -237,7 +237,7 @@ By default, the performance statistics are plotted on their actual scale and the * With `deltas = "mixed"`, the performance statistics (i.e., their point estimates) are plotted on the actual scale, but the uncertainty bars visualize the difference-scale uncertainty. Since the difference-scale uncertainty is usually more helpful than the actual-scale uncertainty (at least with regard to the decision for a final submodel size), we plot with `deltas = TRUE` here (`deltas = "mixed"` would be another good choice). We also set `show_cv_proportions = TRUE` (via its global option) for illustrative purposes, which becomes clearer in section ["Predictor ranking(s) from final `cv_varsel()` run and identification of the selected submodel"](#rksel): -```{r plot_vsel} +```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 9, with uncertainty intervals. The x-axis labels show the size, the predictor that the full-data search added at that size, and its ranking proportion over the CV folds. A dashed horizontal line at zero marks the reference model. The difference comes close to zero and levels off from size 7 on."} options(projpred.plot_vsel_show_cv_proportions = TRUE) plot(cvvs, stats = "mlpd", deltas = TRUE) ``` @@ -313,7 +313,7 @@ This is the same as column `ranking_fulldata` in the `summary.vsel()` output abo Note that we have cut off the search at `nterms_max = 9` (which is smaller than the number of predictor terms in the full model, 20 here), so the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). The *transposed* matrix of ranking proportions can be visualized via `plot.cv_proportions()`: -```{r plot_cv_proportions} +```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking, the x-axis shows the ranking positions 1 to 9, and the color and the number of each tile give the proportion of CV folds that have the predictor at that position."} plot(pr_rk) ``` Apart from visualizing the variability in the ranking of the predictors (here, this is of little use because of `K = 2`), this plot will be helpful later. @@ -323,7 +323,7 @@ To retrieve the predictor terms of the final submodel (except for the intercept ( predictors_final <- head(rk[["fulldata"]], size_decided) ) ``` At this place, it is again helpful to take the ranking proportions into account, but now in a cumulated fashion: -```{r plot_cv_proportions_cumul} +```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each position j (x-axis, labeled <=j), the color and the number of the tile give the proportion of CV folds that have the predictor among their first j predictors. In the column <=7, the tiles of the first 7 predictors are at 100 %."} plot(cv_proportions(rk, cumulate = TRUE)) ``` This plot shows that the two fold-wise searches (as well as the full-data search, whose predictor ranking determines the order of the predictors on the y-axis) agree on the *set* of the 7 most relevant predictors: When looking at `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. @@ -400,7 +400,7 @@ A visualization of the projected posterior can be achieved with the [**bayesplot ```{r bayesplot_attach, message=FALSE} library(bayesplot) ``` -```{r bayesplot_prj} +```{r bayesplot_prj, fig.alt="Interval plot of the marginals of the projected posterior for the intercept, the coefficients of the 7 selected predictors and sigma. Each parameter has a point for the median, a thick line for the 50 % interval and a thin line for the 90 % interval."} bayesplot_theme_set(ggplot2::theme_bw()) mcmc_intervals(prj_mat) + ggplot2::coord_cartesian(xlim = c(-1.5, 1.6)) @@ -410,7 +410,7 @@ Note that we only visualize the *1-dimensional* marginals of the projected poste To gain a more complete picture, we would have to visualize at least some *2-dimensional* marginals of the projected posterior (i.e., marginals for pairs of parameters). For comparison, consider the marginal posteriors of the corresponding parameters in the reference model: -```{r bayesplot_ref} +```{r bayesplot_ref, fig.alt="The same interval plot for the marginal posteriors of the same parameters in the reference model. The intervals differ only slightly from those of the projected posterior."} refm_mat <- as.matrix(refm_fit) mcmc_intervals(refm_mat, pars = colnames(prj_mat)) + ggplot2::coord_cartesian(xlim = c(-1.5, 1.6)) @@ -443,7 +443,7 @@ If `dat_gauss_new` also contained response values (i.e., `y` values in this exam With `proj_predict()`, we can obtain draws from predictive distributions based on the final submodel. In contrast to `proj_linpred(<...>, integrated = FALSE)`, this encompasses not only the uncertainty arising from parameter estimation, but also the uncertainty arising from the observation (or "sampling") model for the response^[In case of the Gaussian family we are using here, the uncertainty arising from the observation model is the uncertainty due to the residual standard deviation.]. This is useful for what is usually termed a posterior predictive check (PPC), but would have to be termed something like a posterior-projection predictive check (PPPC) here: -```{r proj_predict} +```{r proj_predict, fig.alt="Density overlay plot: the density of the observed response y as a dark line, and the densities of the predictive draws from proj_predict() as many light lines. The light lines are close to the dark line."} prj_predict <- proj_predict(prj) # Using the 'bayesplot' package: ppc_dens_overlay(y = dat_gauss$y, yrep = prj_predict) From afb333b5ed532db84ae258df1e0285bfadfbb347 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Sun, 15 Nov 2026 23:44:24 +0200 Subject: [PATCH 02/40] Describe the shape of the two GMPD plots in their alt text --- vignettes/latent.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 573ed784e..93f1d1f18 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -201,7 +201,7 @@ In order to decide for a submodel size, we first inspect the `plot()` results. In contrast to the main vignette where we used the mean log predictive density (MLPD) as predictive performance statistic for a `gaussian()` family reference model (and `gaussian()` submodels), we have a discrete family (`poisson()`) here, so it makes sense to exponentiate the MLPD to obtain the geometric mean predictive density (GMPD; in case of a discrete response, the predictive density values are actually predictive *probabilities* and hence the GMPD is bounded by 0 and 1). As in the main vignette, we plot with `deltas = TRUE` (in case of the GMPD, this means that the *ratio* of the submodel GMPD vs. the reference model GMPD is shown). Via global option `projpred.plot_vsel_size_position`, we set argument `size_position` of `plot.vsel()` to `"secondary_x"` to make the submodel sizes readable in all of the plots in this vignette. -```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to the GMPD of the reference model against the submodel size, with uncertainty intervals, for the latent projection of the Poisson model. A dashed horizontal line at 1 marks the reference model."} +```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to the GMPD of the reference model against the submodel size, with uncertainty intervals, for the latent projection of the Poisson model. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 up to size 10 and is above 1 from size 11 on."} options(projpred.plot_vsel_size_position = "secondary_x") ( gg_lat <- plot(vs_lat, stats = "gmpd", deltas = TRUE) ) ``` @@ -401,7 +401,7 @@ Again, the message telling that `$dis` consists of only `NA`s will not The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin, fig.alt="Plot of the GMPD ratio of each submodel to the reference model against the submodel size, with uncertainty intervals, for the latent projection of the negative binomial model. A dashed horizontal line at 1 marks the reference model."} +```{r plot_vsel_nebin, fig.alt="Plot of the GMPD ratio of each submodel to the reference model against the submodel size, with uncertainty intervals, for the latent projection of the negative binomial model. A dashed horizontal line at 1 marks the reference model. The ratio increases with the size and levels off from size 11 on."} ( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) ``` From a9da1f12f75c33ff223c4573f2ba23bf368da6a2 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Sun, 15 Nov 2026 23:49:25 +0200 Subject: [PATCH 03/40] Name the axes in the alt texts and keep their result claims to the vignette text --- vignettes/latent.Rmd | 14 +++++++------- vignettes/projpred.Rmd | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 93f1d1f18..beea5fdc5 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -201,7 +201,7 @@ In order to decide for a submodel size, we first inspect the `plot()` results. In contrast to the main vignette where we used the mean log predictive density (MLPD) as predictive performance statistic for a `gaussian()` family reference model (and `gaussian()` submodels), we have a discrete family (`poisson()`) here, so it makes sense to exponentiate the MLPD to obtain the geometric mean predictive density (GMPD; in case of a discrete response, the predictive density values are actually predictive *probabilities* and hence the GMPD is bounded by 0 and 1). As in the main vignette, we plot with `deltas = TRUE` (in case of the GMPD, this means that the *ratio* of the submodel GMPD vs. the reference model GMPD is shown). Via global option `projpred.plot_vsel_size_position`, we set argument `size_position` of `plot.vsel()` to `"secondary_x"` to make the submodel sizes readable in all of the plots in this vignette. -```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to the GMPD of the reference model against the submodel size, with uncertainty intervals, for the latent projection of the Poisson model. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 up to size 10 and is above 1 from size 11 on."} +```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the latent projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 below size 11 and reaches the reference model at size 11."} options(projpred.plot_vsel_size_position = "secondary_x") ( gg_lat <- plot(vs_lat, stats = "gmpd", deltas = TRUE) ) ``` @@ -258,7 +258,7 @@ time_trad <- system.time(vs_trad <- varsel( options(warn_instable_orig) rm(warn_instable_orig) ``` -```{r post_vs_trad, fig.alt="The same plot for the traditional projection of the Poisson model. Several noise terms enter the predictor ranking before relevant ones, and the submodels reach the GMPD of the reference model only at submodel size 28."} +```{r post_vs_trad, fig.alt="The same plot for the traditional projection. Several noise terms enter the predictor ranking before relevant ones, and the submodels reach the reference model only at size 28."} print(time_trad) ( gg_trad <- plot(vs_trad, stats = "gmpd", deltas = TRUE) ) ``` @@ -401,7 +401,7 @@ Again, the message telling that `$dis` consists of only `NA`s will not The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin, fig.alt="Plot of the GMPD ratio of each submodel to the reference model against the submodel size, with uncertainty intervals, for the latent projection of the negative binomial model. A dashed horizontal line at 1 marks the reference model. The ratio increases with the size and levels off from size 11 on."} +```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size and reaches the reference model at size 10 or 11. Up to size 11, the predictor ranking includes the noise term xn.29 and does not include x.9."} ( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) ``` @@ -587,13 +587,13 @@ In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values bec We will also ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. Plot the results: -```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size, with uncertainty intervals, for the Weibull model with right-censored observations. A dashed horizontal line at zero marks the reference model."} +```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model, with uncertainty bars, for the Weibull model with right-censored observations. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 7 on."} plot(cvvs_weib, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. Project onto the submodel consisting of the first `n_pred_truth` predictors and perform a "posterior-projection predictive check" (PPPC): -```{r weibull_pppc, fig.alt="Kaplan-Meier overlay plot for the Weibull submodel: the Kaplan-Meier estimate of the survival function of the observed times as a dark line, and the Kaplan-Meier estimates of the predictive draws from proj_predict() as light lines. The light lines are close to the dark line."} +```{r weibull_pppc, fig.alt="Kaplan-Meier overlay plot with the time on the x-axis and the survival probability on the y-axis: the Kaplan-Meier curve of the observed times as a dark line, and the Kaplan-Meier curves of predictive draws from the Weibull submodel as light lines. The light lines are close to the dark line."} predictors_final_weib <- head(ranking(cvvs_weib)[["fulldata"]], n_pred_truth) prj_weib <- project(refm_weib, predictor_terms = predictors_final_weib) prj_predict_weib <- proj_predict(prj_weib) @@ -730,13 +730,13 @@ In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values bec We will also ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. Plot the results: -```{r lognormal_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size, with uncertainty intervals, for the log-normal model with right-censored observations. A dashed horizontal line at zero marks the reference model."} +```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 8 on."} plot(cvvs_lnorm, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. Project onto the submodel consisting of the first `n_pred_truth` predictors and perform a "posterior-projection predictive check" (PPPC): -```{r lognormal_pppc, fig.alt="Kaplan-Meier overlay plot for the log-normal submodel: the Kaplan-Meier estimate of the survival function of the observed times as a dark line, and the Kaplan-Meier estimates of the predictive draws from proj_predict() as light lines. The light lines are close to the dark line."} +```{r lognormal_pppc, fig.alt="The same Kaplan-Meier overlay plot for the log-normal submodel. The light lines are close to the dark line."} predictors_final_lnorm <- head(ranking(cvvs_lnorm)[["fulldata"]], n_pred_truth) prj_lnorm <- project(refm_lnorm, predictor_terms = predictors_final_lnorm) prj_predict_lnorm <- proj_predict(prj_lnorm) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 5fe5de08b..b8dd8c9a4 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -152,7 +152,7 @@ cvvs_fast <- cv_varsel( To find a suitable value for `nterms_max` in subsequent `cv_varsel()` runs, we take a look at a plot of at least one predictive performance statistic in dependence of the submodel size. Here, we choose the mean log predictive density (MLPD; see the documentation for argument `stats` of `summary.vsel()` for details) as the only performance statistic. Since we will be using the following plot only to determine `nterms_max` for subsequent `cv_varsel()` runs, we can omit the predictor ranking from the plot by setting `ranking_nterms_max` to `NA` (which requires `size_position = "primary_x_bottom"`, which we set here via its global option, just like argument `text_angle`): -```{r plot_vsel_fast, fig.alt="Line plot of the mean log predictive density (MLPD) of the submodels against the submodel size from 0 to 20, from the preliminary L1 search with refit_prj = FALSE. A dashed horizontal line shows the MLPD of the reference model. The submodel MLPD increases with the size and levels off from size 8 on."} +```{r plot_vsel_fast, fig.alt="Plot of the mean log predictive density (MLPD) of the submodels against the submodel size from 0 to 20, with uncertainty bars, from the preliminary L1 search without refitted projections. A dashed horizontal line marks the MLPD of the reference model. The submodel MLPD increases with the size and levels off from size 8 on."} options(projpred.plot_vsel_size_position = "primary_x_bottom") options(projpred.plot_vsel_text_angle = 0) plot(cvvs_fast, stats = "mlpd", ranking_nterms_max = NA) @@ -176,7 +176,7 @@ cvvs_fast_refit <- cv_varsel( Here, we ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. With the `refit_prj = TRUE` results, the predictive performance plot from above now looks as follows: -```{r plot_vsel_fast_refit, fig.alt="The same plot as the previous one, from the preliminary run with refit_prj = TRUE. The submodel MLPD changes little after size 8."} +```{r plot_vsel_fast_refit, fig.alt="The same plot from the preliminary run with refitted projections. The submodel MLPD changes little after size 8."} plot(cvvs_fast_refit, stats = "mlpd", ranking_nterms_max = NA) ``` This refined plot confirms that the submodel MLPD does not change much after submodel size 8, so in our final `cv_varsel()` run, we set `nterms_max` to a value slightly higher than 8 (here: 9) to ensure that we see the MLPD leveling off. @@ -237,7 +237,7 @@ By default, the performance statistics are plotted on their actual scale and the * With `deltas = "mixed"`, the performance statistics (i.e., their point estimates) are plotted on the actual scale, but the uncertainty bars visualize the difference-scale uncertainty. Since the difference-scale uncertainty is usually more helpful than the actual-scale uncertainty (at least with regard to the decision for a final submodel size), we plot with `deltas = TRUE` here (`deltas = "mixed"` would be another good choice). We also set `show_cv_proportions = TRUE` (via its global option) for illustrative purposes, which becomes clearer in section ["Predictor ranking(s) from final `cv_varsel()` run and identification of the selected submodel"](#rksel): -```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 9, with uncertainty intervals. The x-axis labels show the size, the predictor that the full-data search added at that size, and its ranking proportion over the CV folds. A dashed horizontal line at zero marks the reference model. The difference comes close to zero and levels off from size 7 on."} +```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 9, with uncertainty bars. Below each size, the x-axis shows the predictor that the full-data search added and the proportion of CV folds that put this predictor at the same position. A dashed horizontal line at zero marks the reference model. The difference comes close to zero and levels off from size 7 on."} options(projpred.plot_vsel_show_cv_proportions = TRUE) plot(cvvs, stats = "mlpd", deltas = TRUE) ``` @@ -313,7 +313,7 @@ This is the same as column `ranking_fulldata` in the `summary.vsel()` output abo Note that we have cut off the search at `nterms_max = 9` (which is smaller than the number of predictor terms in the full model, 20 here), so the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). The *transposed* matrix of ranking proportions can be visualized via `plot.cv_proportions()`: -```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking, the x-axis shows the ranking positions 1 to 9, and the color and the number of each tile give the proportion of CV folds that have the predictor at that position."} +```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking and the x-axis the ranking positions 1 to 9. Each tile gives the proportion of CV folds that have the predictor at that position."} plot(pr_rk) ``` Apart from visualizing the variability in the ranking of the predictors (here, this is of little use because of `K = 2`), this plot will be helpful later. @@ -323,7 +323,7 @@ To retrieve the predictor terms of the final submodel (except for the intercept ( predictors_final <- head(rk[["fulldata"]], size_decided) ) ``` At this place, it is again helpful to take the ranking proportions into account, but now in a cumulated fashion: -```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each position j (x-axis, labeled <=j), the color and the number of the tile give the proportion of CV folds that have the predictor among their first j predictors. In the column <=7, the tiles of the first 7 predictors are at 100 %."} +```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 9 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At size 7, the tiles of the first 7 predictors are at 100 %."} plot(cv_proportions(rk, cumulate = TRUE)) ``` This plot shows that the two fold-wise searches (as well as the full-data search, whose predictor ranking determines the order of the predictors on the y-axis) agree on the *set* of the 7 most relevant predictors: When looking at `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. @@ -400,7 +400,7 @@ A visualization of the projected posterior can be achieved with the [**bayesplot ```{r bayesplot_attach, message=FALSE} library(bayesplot) ``` -```{r bayesplot_prj, fig.alt="Interval plot of the marginals of the projected posterior for the intercept, the coefficients of the 7 selected predictors and sigma. Each parameter has a point for the median, a thick line for the 50 % interval and a thin line for the 90 % interval."} +```{r bayesplot_prj, fig.alt="Interval plot of the projected posterior, with one row for the intercept, each of the 7 selected predictors and sigma, and the parameter value on the x-axis. Each row has a point for the median, a thick line for the 50 % interval and a thin line for the 90 % interval."} bayesplot_theme_set(ggplot2::theme_bw()) mcmc_intervals(prj_mat) + ggplot2::coord_cartesian(xlim = c(-1.5, 1.6)) @@ -410,7 +410,7 @@ Note that we only visualize the *1-dimensional* marginals of the projected poste To gain a more complete picture, we would have to visualize at least some *2-dimensional* marginals of the projected posterior (i.e., marginals for pairs of parameters). For comparison, consider the marginal posteriors of the corresponding parameters in the reference model: -```{r bayesplot_ref, fig.alt="The same interval plot for the marginal posteriors of the same parameters in the reference model. The intervals differ only slightly from those of the projected posterior."} +```{r bayesplot_ref, fig.alt="The same interval plot for the posterior of the same parameters in the reference model. The intervals differ only slightly from those of the projected posterior."} refm_mat <- as.matrix(refm_fit) mcmc_intervals(refm_mat, pars = colnames(prj_mat)) + ggplot2::coord_cartesian(xlim = c(-1.5, 1.6)) @@ -443,7 +443,7 @@ If `dat_gauss_new` also contained response values (i.e., `y` values in this exam With `proj_predict()`, we can obtain draws from predictive distributions based on the final submodel. In contrast to `proj_linpred(<...>, integrated = FALSE)`, this encompasses not only the uncertainty arising from parameter estimation, but also the uncertainty arising from the observation (or "sampling") model for the response^[In case of the Gaussian family we are using here, the uncertainty arising from the observation model is the uncertainty due to the residual standard deviation.]. This is useful for what is usually termed a posterior predictive check (PPC), but would have to be termed something like a posterior-projection predictive check (PPPC) here: -```{r proj_predict, fig.alt="Density overlay plot: the density of the observed response y as a dark line, and the densities of the predictive draws from proj_predict() as many light lines. The light lines are close to the dark line."} +```{r proj_predict, fig.alt="Density overlay plot with the response value on the x-axis: the density of the observed response as a dark line, and the densities of predictive draws from the submodel as many light lines. The light lines are close to the dark line."} prj_predict <- proj_predict(prj) # Using the 'bayesplot' package: ppc_dens_overlay(y = dat_gauss$y, yrep = prj_predict) From 7a6da7a2b9576e9e4be29adf83e75196d22e683f Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 12:30:36 +0200 Subject: [PATCH 04/40] Use cv_varsel() with LOO CV and validated search in the main vignette --- vignettes/projpred.Rmd | 161 ++++++++++++++--------------------------- 1 file changed, 53 insertions(+), 108 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index a316f2998..b023a13b6 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -127,99 +127,34 @@ There are two functions for running the combination of search and evaluation: `v In contrast to `varsel()`, `cv_varsel()` performs a cross-validation (CV). With `cv_method = "LOO"` (the default), `cv_varsel()` runs a Pareto-smoothed importance sampling leave-one-out CV [PSIS-LOO CV, see @vehtari_practical_2017; @vehtari_pareto_2022]. With `cv_method = "kfold"`, `cv_varsel()` runs a $K$-fold CV. The extent of the CV mainly depends on `cv_varsel()`'s argument `validate_search`: If `validate_search = TRUE` (the default), the search part is run with the training data of each CV fold separately and the evaluation part is run with the corresponding test data of each CV fold. If `validate_search = FALSE`, the search is excluded from the CV so that only a single full-data search is run. Because of its more thorough protection against overfitting^[Currently, neither `varsel()` nor `cv_varsel()` (not even `cv_varsel()` with `validate_search = TRUE`) guard against overfitting in the selection of the submodel *size*. This is why we added "approximately" to "valid post-selection inference" in section ["Introduction"](#intro). Typically, however, the overfitting induced by the size selection should be comparatively small [@piironen_comparison_2017].], `cv_varsel()` with `validate_search = TRUE` is recommended over `varsel()` and `cv_varsel()` with `validate_search = FALSE`. -Nonetheless, a preliminary and comparatively fast run of `varsel()` or `cv_varsel()` with `validate_search = FALSE` can give a rough idea of the performance of the submodels and can be used for finding a suitable value for argument `nterms_max` in subsequent runs (argument `nterms_max` imposes a limit on the submodel size up to which the search is continued and is thus able to reduce the runtime considerably). +For large models, a preliminary run of `varsel()` or `cv_varsel()` with `validate_search = FALSE` can give a rough idea of the performance of the submodels and help to choose argument `nterms_max` (the largest submodel size of the search) for the final run (see section ["Speed"](#speed)). -### Preliminary `cv_varsel()` run +### `cv_varsel()` run -To illustrate a preliminary `cv_varsel()` run with `validate_search = FALSE`, we set `nterms_max` to the number of predictor terms in the full model, i.e., `nterms_max = 20`. -To speed up the building of the vignette (this is not recommended in general), we choose the `"L1"` search `method` and set `refit_prj` to `FALSE`. -```{r cv_varsel_fast} -# Preliminary cv_varsel() run: -cvvs_fast <- cv_varsel( - refm_obj, - validate_search = FALSE, - ### Only for the sake of speed (not recommended in general): - method = "L1", - refit_prj = FALSE, - ### - nterms_max = 20, - ### In interactive use, we recommend not to deactivate the verbose mode: - verbose = 0 - ### -) -``` - -To find a suitable value for `nterms_max` in subsequent `cv_varsel()` runs, we take a look at a plot of at least one predictive performance statistic in dependence of the submodel size. -Here, we choose the mean log predictive density (MLPD; see the documentation for argument `stats` of `summary.vsel()` for details) as the only performance statistic. -Since we will be using the following plot only to determine `nterms_max` for subsequent `cv_varsel()` runs, we can omit the predictor ranking from the plot by setting `ranking_nterms_max` to `NA` (which requires `size_position = "primary_x_bottom"`, which we set here via its global option, just like argument `text_angle`): -```{r plot_vsel_fast, fig.alt="Plot of the mean log predictive density (MLPD) of the submodels against the submodel size from 0 to 20, with uncertainty bars, from the preliminary L1 search without refitted projections. A dashed horizontal line marks the MLPD of the reference model. The submodel MLPD increases with the size and levels off from size 8 on."} -options(projpred.plot_vsel_size_position = "primary_x_bottom") -options(projpred.plot_vsel_text_angle = 0) -plot(cvvs_fast, stats = "mlpd", ranking_nterms_max = NA) -``` -This plot suggests that the submodel MLPD levels off from submodel size 8 on. -However, we used L1 search with `refit_prj = FALSE`, which means that the projections employed for the predictive performance evaluation are L1-penalized, which is usually undesired [@piironen_projective_2020, section 4]. -Thus, to investigate the impact of `refit_prj = FALSE`, we re-run `cv_varsel()`, but this time with the default of `refit_prj = TRUE` and re-using the search results (as well as the CV-related arguments such as `validate_search`; see section "Usage" in `?cv_varsel.vsel`) from the former `cv_varsel()` call (this is done by applying `cv_varsel()` to `cvvs_fast` instead of `refm_obj` so that the `cv_varsel()` generic dispatches to the `cv_varsel.vsel()` method that was introduced in **projpred** 2.8.0^[Analogous functionality has been implemented for `varsel()`, namely in `varsel.vsel()`.]). -To save time, we also set `nclusters_pred` to a comparatively low value of `20`: -```{r cv_varsel_fast_refit} -# Preliminary cv_varsel() run with `refit_prj = TRUE`: -cvvs_fast_refit <- cv_varsel( - cvvs_fast, - ### Only for the sake of speed (not recommended in general): - nclusters_pred = 20, - ### - ### In interactive use, we recommend not to deactivate the verbose mode: - verbose = 0 - ### -) -``` -Here, we ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. - -With the `refit_prj = TRUE` results, the predictive performance plot from above now looks as follows: -```{r plot_vsel_fast_refit, fig.alt="The same plot from the preliminary run with refitted projections. The submodel MLPD changes little after size 8."} -plot(cvvs_fast_refit, stats = "mlpd", ranking_nterms_max = NA) -``` -This refined plot confirms that the submodel MLPD does not change much after submodel size 8, so in our final `cv_varsel()` run, we set `nterms_max` to a value slightly higher than 8 (here: 9) to ensure that we see the MLPD leveling off. -The search results from the initial `cvvs_fast` object could now be re-used again (via `cv_varsel.vsel()`) for investigating the sensitivity of the results to changes in `nclusters_pred` (or `ndraws_pred`). -Here, we skip this for the sake of brevity and instead head over to the final `cv_varsel()` run. +Here, we run `cv_varsel()` with its defaults: PSIS-LOO CV (`cv_method = "LOO"`), validated search (`validate_search = TRUE`) and forward search (`method = "forward"`). +For a Gaussian reference model like this one, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix, so this run takes only a few seconds. +In LOO CV with validated search, the search of each fold uses random observation weights (see argument `loo_perturb_alpha` of `cv_varsel()`). +This makes the estimated predictive performance change more smoothly between submodel sizes. -### Final `cv_varsel()` run +The code below also shows how **projpred**'s CV can be parallelized with the **future** package (we disable the parallelization for the vignette build). +Each LOO fold is then one task for the parallel workers. +For a model as small as this one, the parallelization overhead is larger than the gain. -For this final `cv_varsel()` run (with `validate_search = TRUE`, as recommended), we use a $K$-fold CV with a small number of folds (`K = 2`) to make this vignette build faster. -In practice, we recommend using either the default of `cv_method = "LOO"` (possibly subsampled, see argument `nloo` of `cv_varsel()`) or a larger value for `K` if this is possible in terms of computation time. -Here, we also perform the $K$ reference model refits outside of `cv_varsel()`. -Although not strictly necessary here, this is helpful in practice because often, `cv_varsel()` needs to be re-run multiple times in order to try out different argument settings. -We also show how **projpred**'s CV (i.e., the CV comprising search and performance evaluation, after refitting the reference model $K$ times) can be parallelized (though we disable it here for vignette building). -In practice, parallelization is of little use with only `K = 2` folds and fast fold-wise operations, as the parallelization overhead eats up any runtime improvements. -Note that for this final `cv_varsel()` run, we cannot make use of `cv_varsel.vsel()` (by applying `cv_varsel()` to `cvvs_fast` or `cvvs_fast_refit` instead of `refm_obj`) because of the change in `nterms_max` (here: 9, for `cvvs_fast` and `cvvs_fast_refit`: 20). +If PSIS-LOO CV is not reliable (for example, if many Pareto $\hat{k}$ values are high), $K$-fold CV (`cv_method = "kfold"`) is an alternative. +Then `run_cvfun()` can run the $K$ reference model refits once, so that several `cv_varsel()` calls can use them (see argument `cvfits` of `cv_varsel()`). ```{r cv_varsel, message=FALSE} -# Refit the reference model K times: -cv_fits <- run_cvfun( - refm_obj, - ### Only for the sake of speed (not recommended in general): - K = 2 - ### -) # For running projpred's CV in parallel (see cv_varsel()'s argument `parallel`): # Note: Parallel processing is disabled during package building to avoid issues use_parallel <- FALSE # Set to TRUE for actual parallel processing if (use_parallel) { future::plan(future::multisession, workers = ncores) } -# Final cv_varsel() run: cvvs <- cv_varsel( refm_obj, - cv_method = "kfold", - cvfits = cv_fits, - ### Only for the sake of speed (not recommended in general): - method = "L1", - nclusters_pred = 20, - ### - nterms_max = 9, parallel = use_parallel, ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0 - ### + ### ) # Tear down the CV parallelization setup: if (use_parallel) { @@ -227,23 +162,26 @@ if (use_parallel) { } ``` -#### Predictive performance plot from final `cv_varsel()` run {#plotfinal} +#### Predictive performance plot from the `cv_varsel()` run {#plotfinal} -We can now select a final submodel size by looking at a predictive performance plot similar to the one created for the preliminary `cv_varsel()` run above. +We can now select a final submodel size by looking at a predictive performance plot. By default, the performance statistics are plotted on their actual scale and the uncertainty bars match this scale, but argument `deltas` of `plot.vsel()` offers two more options: * With `deltas = TRUE`, the performance statistics are plotted on *difference scale*, i.e., as differences^[For the geometric mean predictive density (GMPD; see argument `stats` of `summary.vsel()` and `plot.vsel()`), `deltas = TRUE` estimates the GMPD *ratio* (not difference) vs. the baseline model.] from the baseline model^[For the definition of the baseline model, see argument `baseline` of `summary.vsel()` and `plot.vsel()`; in the most common cases, the default baseline model is the reference model.] and the uncertainty bars match this scale, * With `deltas = "mixed"`, the performance statistics (i.e., their point estimates) are plotted on the actual scale, but the uncertainty bars visualize the difference-scale uncertainty. -Since the difference-scale uncertainty is usually more helpful than the actual-scale uncertainty (at least with regard to the decision for a final submodel size), we plot with `deltas = TRUE` here (`deltas = "mixed"` would be another good choice). We also set `show_cv_proportions = TRUE` (via its global option) for illustrative purposes, which becomes clearer in section ["Predictor ranking(s) from final `cv_varsel()` run and identification of the selected submodel"](#rksel): -```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 9, with uncertainty bars. Below each size, the x-axis shows the predictor that the full-data search added and the proportion of CV folds that put this predictor at the same position. A dashed horizontal line at zero marks the reference model. The difference comes close to zero and levels off from size 7 on."} +Since the difference-scale uncertainty is usually more helpful than the actual-scale uncertainty (at least with regard to the decision for a final submodel size), we plot with `deltas = TRUE` here (`deltas = "mixed"` would be another good choice). We also set `show_cv_proportions = TRUE` (via its global option) for illustrative purposes, which becomes clearer in section ["Predictor ranking(s) from the `cv_varsel()` run and identification of the selected submodel"](#rksel). +The search ran up to submodel size 19 (the default of argument `nterms_max` of `cv_varsel()` here), but the plot shows only the sizes up to 12 (argument `nterms_max` of `plot.vsel()`) to keep the labels readable: +```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 12, with uncertainty bars. Below each size, the x-axis shows the predictor that the full-data search added and the proportion of CV folds that put this predictor at the same position. A dashed horizontal line at zero marks the reference model. The difference reaches zero at size 7 and stays close to zero for larger sizes."} +options(projpred.plot_vsel_size_position = "primary_x_bottom") +options(projpred.plot_vsel_text_angle = 0) options(projpred.plot_vsel_show_cv_proportions = TRUE) -plot(cvvs, stats = "mlpd", deltas = TRUE) +plot(cvvs, stats = "mlpd", deltas = TRUE, nterms_max = 12) ``` #### Decision for final submodel size {#decision-size} -Based on that final predictive performance plot, we decide for a submodel size. +Based on that predictive performance plot, we decide for a submodel size. Usually, the aim is to find the smallest submodel size where the predictive performance of the submodels levels off and is close enough to the reference model's predictive performance (the dashed red horizontal line). Sometimes, the plot may be ambiguous because after reaching the reference model's performance, the submodels' performance may keep increasing (and hence become even better than the reference model's performance^[In general, only `cv_varsel()` with `validate_search = TRUE` allows to judge whether the submodels perform better than the reference model or not. Such a judgment is generally not possible with `varsel()` or `cv_varsel()` with `validate_search = FALSE`.]). @@ -253,7 +191,7 @@ Here, we decide for a submodel size of 7 because it seems to provide the best tr ```{r size_man} size_decided <- 7 ``` -In section ["Predictor ranking(s) from final `cv_varsel()` run and identification of the selected submodel"](#rksel), the predictor ranking and the (CV) ranking proportions that are shown in the plot (below the submodel sizes on the x-axis) are explained in detail---and also how they could have been incorporated into our decision for a submodel size. +In section ["Predictor ranking(s) from the `cv_varsel()` run and identification of the selected submodel"](#rksel), the predictor ranking and the (CV) ranking proportions that are shown in the plot (below the submodel sizes on the x-axis) are explained in detail---and also how they could have been incorporated into our decision for a submodel size. The `suggest_size()` function offered by **projpred** may help in the decision for a submodel size, but this is a rather heuristic method and needs to be interpreted with caution (see `?suggest_size`): ```{r size_sgg} @@ -261,7 +199,7 @@ suggest_size(cvvs, stat = "mlpd") ``` In this case, that heuristic gives the same final submodel size (`7`) as our manual decision. -#### Predictive performance table from final `cv_varsel()` run +#### Predictive performance table from the `cv_varsel()` run A tabular representation of the plot created by `plot.vsel()` can be achieved via `summary.vsel()` or `performances()`. In contrast to `performances()`, the output of `summary.vsel()` contains more information than just the predictive performance results, which is also why there is a sophisticated `print()` method for objects of class `vselsummary` (the output of `summary.vsel()`). This method `print.vselsummary()` is also called by the shortcut method `print.vsel()` which can be applied to an object resulting from `varsel()` or `cv_varsel()`. @@ -281,15 +219,16 @@ perf <- performances(smmry) str(perf) ``` -#### Predictor ranking(s) from final `cv_varsel()` run and identification of the selected submodel {#rksel} +#### Predictor ranking(s) from the `cv_varsel()` run and identification of the selected submodel {#rksel} As indicated by its column name, the predictor ranking from column `ranking_fulldata` of the `summary.vsel()` output is based on the full-data search. -This full-data predictor ranking is also what is shown in the second line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from final `cv_varsel()` run"](#plotfinal). +This full-data predictor ranking is also what is shown in the second line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from the `cv_varsel()` run"](#plotfinal). In case of `cv_varsel()` with `validate_search = TRUE`, there is not only the full-data search, but also fold-wise searches, implying that there are also fold-wise predictor rankings. -All of these predictor rankings (the full-data one and---if available---the fold-wise ones) can be retrieved via `ranking()`: +All of these predictor rankings (the full-data one and---if available---the fold-wise ones) can be retrieved via `ranking()`. +As in the plot above, we cut off the rankings at 12 predictor terms (argument `nterms_max` of `ranking()`): ```{r ranking} -rk <- ranking(cvvs) +rk <- ranking(cvvs, nterms_max = 12) ``` In addition to inspecting the full-data predictor ranking, it often makes sense to investigate the ranking proportions derived from the fold-wise predictor rankings (only available in case of `cv_varsel()` with `validate_search = TRUE`, which we have here) in order to get a sense for the variability in the ranking of the predictors. @@ -298,10 +237,11 @@ To compute these ranking proportions, we use `cv_proportions()`: ```{r cv_proportions} ( pr_rk <- cv_proportions(rk) ) ``` -The main diagonal of this matrix is contained in column `cv_proportions_diag` of the `summary.vsel()` output and is also shown in the third line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from final `cv_varsel()` run"](#plotfinal) (due to `show_cv_proportions = TRUE` set via its global option). +The main diagonal of this matrix is contained in column `cv_proportions_diag` of the `summary.vsel()` output and is also shown in the third line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from the `cv_varsel()` run"](#plotfinal) (due to `show_cv_proportions = TRUE` set via its global option). -Here, the ranking proportions are of little use as we have used `K = 2` (in the final `cv_varsel()` call above) for the sake of speed. -Nevertheless, we can see that the two CV folds agree on the most relevant predictor term (`X1`) and the second most relevant predictor term (`X14`). +Here, 93 % of the LOO CV folds have `X1` at position 1 and `X14` at position 2; the other folds have these two predictors in the opposite order. +At positions 3 to 8, at least 96 % of the folds have the same predictor as the full-data predictor ranking. +From position 9 on, the proportions are below 40 %, so the ranking of the remaining predictors varies between the folds. Since the column names of the matrix returned by `cv_proportions()` follow the full-data predictor ranking, we can infer that `X1` and `X14` are also the most relevant predictor terms (in this order) in the full-data predictor ranking. To see this more explicitly, we can access element `fulldata` of the `ranking()` output: ```{r ranking_fulldata} @@ -309,28 +249,27 @@ rk[["fulldata"]] ``` This is the same as column `ranking_fulldata` in the `summary.vsel()` output above (apart from the intercept). -Note that we have cut off the search at `nterms_max = 9` (which is smaller than the number of predictor terms in the full model, 20 here), so the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). +Note that we have cut off the predictor rankings at 12 predictor terms (which is smaller than the number of predictor terms in the full model, 20 here), so the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). The *transposed* matrix of ranking proportions can be visualized via `plot.cv_proportions()`: -```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking and the x-axis the ranking positions 1 to 9. Each tile gives the proportion of CV folds that have the predictor at that position."} +```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking and the x-axis the ranking positions 1 to 12. Each tile gives the proportion of CV folds that have the predictor at that position."} plot(pr_rk) ``` -Apart from visualizing the variability in the ranking of the predictors (here, this is of little use because of `K = 2`), this plot will be helpful later. +Apart from visualizing the variability in the ranking of the predictors, this plot will be helpful later. To retrieve the predictor terms of the final submodel (except for the intercept which is always included in the submodels), we combine the chosen submodel size of 7 with the full-data predictor ranking: ```{r predictors_final} ( predictors_final <- head(rk[["fulldata"]], size_decided) ) ``` At this place, it is again helpful to take the ranking proportions into account, but now in a cumulated fashion: -```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 9 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At size 7, the tiles of the first 7 predictors are at 100 %."} +```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 12 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At sizes 7 and 8, the tiles of the first 7 or 8 predictors are at 100 %."} plot(cv_proportions(rk, cumulate = TRUE)) ``` -This plot shows that the two fold-wise searches (as well as the full-data search, whose predictor ranking determines the order of the predictors on the y-axis) agree on the *set* of the 7 most relevant predictors: When looking at `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. -(Similarly, the two CV folds also agree on the set of the most relevant predictor---which is a singleton---and the set of the two most relevant predictors, but we already observed this above.) +This plot shows that all fold-wise searches (as well as the full-data search, whose predictor ranking determines the order of the predictors on the y-axis) agree on the *set* of the 7 most relevant predictors: When looking at `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. +The same holds for the set of the 8 most relevant predictors (`<=8`), but not for larger sets: for example, only 35 % of the folds have `X10` among their first 9 predictors. -Although not demonstrated here, the cumulated ranking proportions also could have guided the decision for a submodel size: -From the plot of the cumulated ranking proportions, we can see that size 8 might have been an unfortunate choice because `X11` (which---by cutting off the full-data predictor ranking at size 8---would then have been selected as the 8^th^ predictor in the final submodel) is not included among the first 8 terms by any CV fold. -However, since `K = 2` is too small for reliable statements regarding the variability of the predictor ranking, we did not take the cumulated ranking proportions into account when we made our decision for a submodel size above. +The cumulated ranking proportions can also guide the decision for a submodel size. +Here, they support our choice of size 7: the set of selected predictors is stable up to size 8, whereas a size of 9 or more would add a predictor that most folds do not select at that position. In a real-world application, we might also be able to incorporate the full-data predictor ranking into our decision for a submodel size (usually, this requires to also take into account the variability of the predictor ranking, as reflected by the---possibly cumulated---ranking proportions). For example, the predictors might be associated with different measurement costs, so that we might want to select a costly predictor only if the submodel size at which it would be selected (according to the full-data predictor ranking, but taking into account that there might be variability in the ranking of the predictors) comes with a considerable increase in predictive performance. @@ -524,7 +463,8 @@ refm_fit <- stan_gamm4( ) ``` -In case of multilevel models (currently only non-additive ones), **projpred** has two global options that may be relevant for users: `projpred.mlvl_pred_new` and `projpred.mlvl_proj_ref_new`. +In case of multilevel models, **projpred** has two global options that may be relevant for users: `projpred.mlvl_pred_new` and `projpred.mlvl_proj_ref_new`. +For additive multilevel models, `projpred.mlvl_pred_new` also applies. These are explained in detail in the general package documentation (available [online](https://mc-stan.org/projpred/reference/projpred-package.html) or by typing `` ?`projpred-package` ``). ## Troubleshooting @@ -565,14 +505,19 @@ For multilevel models, the augmented-data projection seems to suffer from the sa By suboptimal results, we mean that the relevance of the group-level terms can be underestimated. According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in such cases. -### Speed +### Speed {#speed} + +For Gaussian models with the identity link and for the latent projection, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix, so the defaults of `cv_varsel()` are usually fast for these models. +For other families, each projection is an iterative fit, and `cv_varsel()` with the defaults can take much longer. +For binomial and Poisson models, the latent projection is then an alternative that is often much faster (see the [latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html)). -There are many ways to speed up **projpred**, but in general, such speed-ups lead to results that are less accurate and hence should only be considered as giving *preliminary* results. +There are many other ways to speed up **projpred**, but in general, such speed-ups lead to results that are less accurate and hence should only be considered as giving *preliminary* results. Some speed-up possibilities are: 1. Using `cv_varsel()` with `validate_search = FALSE` instead of `validate_search = TRUE`. In case of `cv_method = "LOO"`^[In case of `cv_method = "kfold"`, the runtime advantage of `validate_search = FALSE` compared to `validate_search = TRUE` is by far not as large as in case of `cv_method = "LOO"`.], `cv_varsel()` with `validate_search = FALSE` has comparable runtime to `varsel()`, but accounts for some overfitting, namely that induced by `varsel()`'s in-sample predictions during the predictive performance evaluation. However, as explained in section ["Variable selection"](#variableselection) (see also section ["Overfitting"](#overfitting)), `cv_varsel()` with `validate_search = FALSE` is more prone to overfitting than `cv_varsel()` with `validate_search = TRUE`. + For Gaussian models and the latent projection, `validate_search = TRUE` is usually fast enough (see above). 1. Using `cv_varsel()` with subsampled PSIS-LOO CV [@magnusson_leave-one-out_2020], see argument `nloo` of `cv_varsel()`. @@ -603,14 +548,14 @@ Some speed-up possibilities are: In case of L1 search, this means that the L1-penalized projections of the regression coefficients are used for the predictive performance evaluation, which is usually undesired [@piironen_projective_2020, section 4]. In case of forward search, this issue does not exist. -1. Parallelizing costly parts of the CV implied by `cv_varsel()` (this was demonstrated in the example above; see argument `parallel` of `cv_varsel()`). +1. Parallelizing the CV folds of `cv_varsel()` with the **future** package (see the example above and argument `parallel` of `cv_varsel()`). + The parallel workers are set up with `future::plan()`, for example `future::plan(future::multisession, workers = 4)`. When using `project()`, parallelizing the projection might also help (see the general package documentation available [online](https://mc-stan.org/projpred/reference/projpred-package.html) or by typing `` ?`projpred-package` ``). 1. Using `varsel.vsel()` or `cv_varsel.vsel()` to re-use previous search results for new performance evaluation(s). This is helpful if the performance evaluation part is run multiple times based on the same search results (e.g., when only arguments `ndraws_pred` or `nclusters_pred` of `varsel()` or `cv_varsel()` are changed). - In the example above, this was illustrated when `cv_varsel()` was applied to `cvvs_fast` instead of `refm_obj` to yield `cvvs_fast_refit`. - In that example, search and performance evaluation were effectively run *separately* by the `cv_varsel()` calls yielding `cvvs_fast` and `cvvs_fast_refit`, respectively. - This is because `cv_varsel()` with `refit_prj = FALSE` (used for `cvvs_fast`) has almost only the computational cost of the search (the performance evaluation with `refit_prj = FALSE` has almost no computational cost) and `cv_varsel.vsel()` (used for `cvvs_fast_refit`) has almost only the computational cost of the performance evaluation (the search in `cv_varsel.vsel()` has no computational cost at all because the previous search results from the `vsel` object are re-used). + For example, `cv_varsel(cvvs, nclusters_pred = 20)` would re-use the searches of the `cvvs` object from above and run only a new performance evaluation. + Thus, a `cv_varsel()` call with `refit_prj = FALSE` (which has almost only the computational cost of the search) followed by `cv_varsel.vsel()` calls (which have only the computational cost of the performance evaluation) runs search and performance evaluation separately. 1. Using `run_cvfun()` in case of repeated $K$-fold CV with the same $K$ reference model refits. The output of `run_cvfun()` is typically used as input for argument `cvfits` of `cv_varsel.refmodel()` (so in order to have a speed improvement, the output of `run_cvfun()` needs to be assigned to an object which is then re-used in multiple `cv_varsel()` calls). From a8bb2c4eebfde482bd0ee4019fae24832cea3c1c Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 12:38:33 +0200 Subject: [PATCH 05/40] Correct the vignette text after review and point the vsel methods to the Speed section --- R/cv_varsel.R | 2 +- R/varsel.R | 2 +- man/cv_varsel.Rd | 2 +- man/varsel.Rd | 2 +- vignettes/projpred.Rmd | 58 +++++++++++++++++++++++++----------------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/R/cv_varsel.R b/R/cv_varsel.R index fea6f3f84..bf495a316 100644 --- a/R/cv_varsel.R +++ b/R/cv_varsel.R @@ -12,7 +12,7 @@ #' is explained in section "Note" below) and by running the evaluation part on #' the corresponding test set of each CV fold. A special method is #' [cv_varsel.vsel()] because it re-uses the search results from an earlier -#' [cv_varsel()] (or [varsel()]) run, as illustrated in the main vignette. +#' [cv_varsel()] (or [varsel()]) run (see section "Speed" of the main vignette). #' #' @inheritParams varsel #' @param cv_method The CV method, either `"LOO"` or `"kfold"`. In the `"LOO"` diff --git a/R/varsel.R b/R/varsel.R index 59cacadd3..eb42d5f85 100644 --- a/R/varsel.R +++ b/R/varsel.R @@ -6,7 +6,7 @@ #' (number of predictor terms). The evaluation part determines the predictive #' performance of the submodels along the predictor ranking. A special method is #' [varsel.vsel()] which re-uses the search results from an earlier -#' [varsel()] (or [cv_varsel()]) run, as illustrated in the main vignette. +#' [varsel()] (or [cv_varsel()]) run (see section "Speed" of the main vignette). #' #' @param object An object of class `refmodel` (returned by [get_refmodel()] or #' [init_refmodel()]) or an object that can be passed to argument `object` of diff --git a/man/cv_varsel.Rd b/man/cv_varsel.Rd index a38fc70d9..d13f551f0 100644 --- a/man/cv_varsel.Rd +++ b/man/cv_varsel.Rd @@ -238,7 +238,7 @@ search part with the training data of each CV fold separately (an exception is explained in section "Note" below) and by running the evaluation part on the corresponding test set of each CV fold. A special method is \code{\link[=cv_varsel.vsel]{cv_varsel.vsel()}} because it re-uses the search results from an earlier -\code{\link[=cv_varsel]{cv_varsel()}} (or \code{\link[=varsel]{varsel()}}) run, as illustrated in the main vignette. +\code{\link[=cv_varsel]{cv_varsel()}} (or \code{\link[=varsel]{varsel()}}) run (see section "Speed" of the main vignette). } \details{ Arguments \code{ndraws}, \code{nclusters}, \code{nclusters_pred}, and \code{ndraws_pred} diff --git a/man/varsel.Rd b/man/varsel.Rd index 458a4abd7..81b0f8972 100644 --- a/man/varsel.Rd +++ b/man/varsel.Rd @@ -171,7 +171,7 @@ known as solution path), i.e., the best submodel for each submodel size (number of predictor terms). The evaluation part determines the predictive performance of the submodels along the predictor ranking. A special method is \code{\link[=varsel.vsel]{varsel.vsel()}} which re-uses the search results from an earlier -\code{\link[=varsel]{varsel()}} (or \code{\link[=cv_varsel]{cv_varsel()}}) run, as illustrated in the main vignette. +\code{\link[=varsel]{varsel()}} (or \code{\link[=cv_varsel]{cv_varsel()}}) run (see section "Speed" of the main vignette). } \details{ Arguments \code{ndraws}, \code{nclusters}, \code{nclusters_pred}, and \code{ndraws_pred} diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index b023a13b6..2ce162d24 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -127,18 +127,20 @@ There are two functions for running the combination of search and evaluation: `v In contrast to `varsel()`, `cv_varsel()` performs a cross-validation (CV). With `cv_method = "LOO"` (the default), `cv_varsel()` runs a Pareto-smoothed importance sampling leave-one-out CV [PSIS-LOO CV, see @vehtari_practical_2017; @vehtari_pareto_2022]. With `cv_method = "kfold"`, `cv_varsel()` runs a $K$-fold CV. The extent of the CV mainly depends on `cv_varsel()`'s argument `validate_search`: If `validate_search = TRUE` (the default), the search part is run with the training data of each CV fold separately and the evaluation part is run with the corresponding test data of each CV fold. If `validate_search = FALSE`, the search is excluded from the CV so that only a single full-data search is run. Because of its more thorough protection against overfitting^[Currently, neither `varsel()` nor `cv_varsel()` (not even `cv_varsel()` with `validate_search = TRUE`) guard against overfitting in the selection of the submodel *size*. This is why we added "approximately" to "valid post-selection inference" in section ["Introduction"](#intro). Typically, however, the overfitting induced by the size selection should be comparatively small [@piironen_comparison_2017].], `cv_varsel()` with `validate_search = TRUE` is recommended over `varsel()` and `cv_varsel()` with `validate_search = FALSE`. -For large models, a preliminary run of `varsel()` or `cv_varsel()` with `validate_search = FALSE` can give a rough idea of the performance of the submodels and help to choose argument `nterms_max` (the largest submodel size of the search) for the final run (see section ["Speed"](#speed)). +For large models, a preliminary run of `varsel()` or `cv_varsel()` with `validate_search = FALSE` can give a rough idea of the performance of the submodels. +It can also help to choose argument `nterms_max` (the largest submodel size of the search) for the run with validated search (see section ["Speed"](#speed)). ### `cv_varsel()` run Here, we run `cv_varsel()` with its defaults: PSIS-LOO CV (`cv_method = "LOO"`), validated search (`validate_search = TRUE`) and forward search (`method = "forward"`). -For a Gaussian reference model like this one, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix, so this run takes only a few seconds. +For a Gaussian reference model with the identity link and without multilevel or additive terms, like this one, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix. +Therefore, this run takes only a few seconds. In LOO CV with validated search, the search of each fold uses random observation weights (see argument `loo_perturb_alpha` of `cv_varsel()`). This makes the estimated predictive performance change more smoothly between submodel sizes. The code below also shows how **projpred**'s CV can be parallelized with the **future** package (we disable the parallelization for the vignette build). -Each LOO fold is then one task for the parallel workers. -For a model as small as this one, the parallelization overhead is larger than the gain. +The LOO folds are then split among the parallel workers. +For a model as small as this one, the parallelization overhead can be larger than the runtime gain. If PSIS-LOO CV is not reliable (for example, if many Pareto $\hat{k}$ values are high), $K$-fold CV (`cv_method = "kfold"`) is an alternative. Then `run_cvfun()` can run the $K$ reference model refits once, so that several `cv_varsel()` calls can use them (see argument `cvfits` of `cv_varsel()`). @@ -171,8 +173,9 @@ By default, the performance statistics are plotted on their actual scale and the * With `deltas = "mixed"`, the performance statistics (i.e., their point estimates) are plotted on the actual scale, but the uncertainty bars visualize the difference-scale uncertainty. Since the difference-scale uncertainty is usually more helpful than the actual-scale uncertainty (at least with regard to the decision for a final submodel size), we plot with `deltas = TRUE` here (`deltas = "mixed"` would be another good choice). We also set `show_cv_proportions = TRUE` (via its global option) for illustrative purposes, which becomes clearer in section ["Predictor ranking(s) from the `cv_varsel()` run and identification of the selected submodel"](#rksel). -The search ran up to submodel size 19 (the default of argument `nterms_max` of `cv_varsel()` here), but the plot shows only the sizes up to 12 (argument `nterms_max` of `plot.vsel()`) to keep the labels readable: -```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 12, with uncertainty bars. Below each size, the x-axis shows the predictor that the full-data search added and the proportion of CV folds that put this predictor at the same position. A dashed horizontal line at zero marks the reference model. The difference reaches zero at size 7 and stays close to zero for larger sizes."} +The search ran up to submodel size 19 (here, the default of argument `nterms_max` of `cv_varsel()`). +To keep the labels readable, the plot shows only the sizes up to 12 (argument `nterms_max` of `plot.vsel()`): +```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 12, with uncertainty bars. Below each size, the x-axis shows the predictor that the full-data search added and the proportion of CV folds that put this predictor at the same position. A dashed horizontal line at zero marks the reference model. The difference reaches zero at size 7 and stays slightly above zero for larger sizes."} options(projpred.plot_vsel_size_position = "primary_x_bottom") options(projpred.plot_vsel_text_angle = 0) options(projpred.plot_vsel_show_cv_proportions = TRUE) @@ -188,10 +191,11 @@ Sometimes, the plot may be ambiguous because after reaching the reference model' In that case, one has to find a suitable trade-off between predictive performance (accuracy) and model size (sparsity) in the context of subject-matter knowledge. Here, we decide for a submodel size of 7 because it seems to provide the best trade-off between sparsity and accuracy (size 7 is the smallest size where the submodel MLPD is close enough to the reference model MLPD and from size 7 on, the submodel MLPD levels off). +At sizes 7 to 10, the submodel MLPD is even slightly higher than the reference model MLPD, which is the case of submodels that perform better than the reference model mentioned above. ```{r size_man} size_decided <- 7 ``` -In section ["Predictor ranking(s) from the `cv_varsel()` run and identification of the selected submodel"](#rksel), the predictor ranking and the (CV) ranking proportions that are shown in the plot (below the submodel sizes on the x-axis) are explained in detail---and also how they could have been incorporated into our decision for a submodel size. +In section ["Predictor ranking(s) from the `cv_varsel()` run and identification of the selected submodel"](#rksel), the predictor ranking and the (CV) ranking proportions that are shown in the plot (below the submodel sizes on the x-axis) are explained in detail, together with how they support our decision for a submodel size. The `suggest_size()` function offered by **projpred** may help in the decision for a submodel size, but this is a rather heuristic method and needs to be interpreted with caution (see `?suggest_size`): ```{r size_sgg} @@ -209,8 +213,9 @@ Specifically, to create the table matching the predictive performance plot above smmry <- summary(cvvs, stats = "mlpd", type = c("mean", "lower", "upper"), - deltas = TRUE) -print(smmry, digits = 1) + deltas = TRUE, + nterms_max = 12) +print(smmry, digits = 2) ``` The generic function `performances()` (with main method `performances.vselsummary()` and the shortcut method `performances.vsel()`) essentially extracts the predictive performance results from the output of `summary.vsel()`: @@ -239,8 +244,9 @@ To compute these ranking proportions, we use `cv_proportions()`: ``` The main diagonal of this matrix is contained in column `cv_proportions_diag` of the `summary.vsel()` output and is also shown in the third line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from the `cv_varsel()` run"](#plotfinal) (due to `show_cv_proportions = TRUE` set via its global option). -Here, 93 % of the LOO CV folds have `X1` at position 1 and `X14` at position 2; the other folds have these two predictors in the opposite order. -At positions 3 to 8, at least 96 % of the folds have the same predictor as the full-data predictor ranking. +Here, 93 % of the LOO CV folds have `X1` at position 1 and `X14` at position 2. +The other folds have these two predictors in the opposite order. +At each of positions 3 to 8, at least 96 % of the folds have the same predictor as the full-data predictor ranking at that position. From position 9 on, the proportions are below 40 %, so the ranking of the remaining predictors varies between the folds. Since the column names of the matrix returned by `cv_proportions()` follow the full-data predictor ranking, we can infer that `X1` and `X14` are also the most relevant predictor terms (in this order) in the full-data predictor ranking. To see this more explicitly, we can access element `fulldata` of the `ranking()` output: @@ -249,7 +255,8 @@ rk[["fulldata"]] ``` This is the same as column `ranking_fulldata` in the `summary.vsel()` output above (apart from the intercept). -Note that we have cut off the predictor rankings at 12 predictor terms (which is smaller than the number of predictor terms in the full model, 20 here), so the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). +We cut off the predictor rankings at 12 predictor terms, which is smaller than the number of predictor terms in the full model (20 here). +Therefore, the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). The *transposed* matrix of ranking proportions can be visualized via `plot.cv_proportions()`: ```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking and the x-axis the ranking positions 1 to 12. Each tile gives the proportion of CV folds that have the predictor at that position."} @@ -262,14 +269,18 @@ To retrieve the predictor terms of the final submodel (except for the intercept ( predictors_final <- head(rk[["fulldata"]], size_decided) ) ``` At this place, it is again helpful to take the ranking proportions into account, but now in a cumulated fashion: -```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 12 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At sizes 7 and 8, the tiles of the first 7 or 8 predictors are at 100 %."} +```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 12 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At sizes 7 and 8, all tiles of the first 7 or 8 predictors are at 100 %, at size 9 only 35 % of the folds include the 9th predictor."} plot(cv_proportions(rk, cumulate = TRUE)) ``` -This plot shows that all fold-wise searches (as well as the full-data search, whose predictor ranking determines the order of the predictors on the y-axis) agree on the *set* of the 7 most relevant predictors: When looking at `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. -The same holds for the set of the 8 most relevant predictors (`<=8`), but not for larger sets: for example, only 35 % of the folds have `X10` among their first 9 predictors. +This plot shows that all fold-wise searches and the full-data search agree on the *set* of the 7 most relevant predictors. +(The full-data predictor ranking determines the order of the predictors on the y-axis.) +At `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. +The same holds for the set of the 8 most relevant predictors (`<=8`), but not for larger sets. +For example, only 35 % of the folds have `X10` among their first 9 predictors. The cumulated ranking proportions can also guide the decision for a submodel size. -Here, they support our choice of size 7: the set of selected predictors is stable up to size 8, whereas a size of 9 or more would add a predictor that most folds do not select at that position. +Here, they agree with our choice of size 7: all folds select the same 7 (and 8) predictors. +A size of 9 would add `X10`, which only 35 % of the folds have among their first 9 predictors. In a real-world application, we might also be able to incorporate the full-data predictor ranking into our decision for a submodel size (usually, this requires to also take into account the variability of the predictor ranking, as reflected by the---possibly cumulated---ranking proportions). For example, the predictors might be associated with different measurement costs, so that we might want to select a costly predictor only if the submodel size at which it would be selected (according to the full-data predictor ranking, but taking into account that there might be variability in the ranking of the predictors) comes with a considerable increase in predictive performance. @@ -464,7 +475,6 @@ refm_fit <- stan_gamm4( ``` In case of multilevel models, **projpred** has two global options that may be relevant for users: `projpred.mlvl_pred_new` and `projpred.mlvl_proj_ref_new`. -For additive multilevel models, `projpred.mlvl_pred_new` also applies. These are explained in detail in the general package documentation (available [online](https://mc-stan.org/projpred/reference/projpred-package.html) or by typing `` ?`projpred-package` ``). ## Troubleshooting @@ -507,9 +517,10 @@ According to the simulation-based case study from [#353](https://github.com/stan ### Speed {#speed} -For Gaussian models with the identity link and for the latent projection, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix, so the defaults of `cv_varsel()` are usually fast for these models. -For other families, each projection is an iterative fit, and `cv_varsel()` with the defaults can take much longer. -For binomial and Poisson models, the latent projection is then an alternative that is often much faster (see the [latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html)). +For Gaussian models with the identity link and without multilevel or additive terms, and for the latent projection without such terms, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix. +Therefore, the defaults of `cv_varsel()` are usually fast for these models. +For other models, each projection is an iterative fit, and `cv_varsel()` with the defaults can take much longer. +For binomial and Poisson models, the latent projection is an alternative that uses the same fast computations (see the [latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html)). There are many other ways to speed up **projpred**, but in general, such speed-ups lead to results that are less accurate and hence should only be considered as giving *preliminary* results. Some speed-up possibilities are: @@ -517,7 +528,7 @@ Some speed-up possibilities are: 1. Using `cv_varsel()` with `validate_search = FALSE` instead of `validate_search = TRUE`. In case of `cv_method = "LOO"`^[In case of `cv_method = "kfold"`, the runtime advantage of `validate_search = FALSE` compared to `validate_search = TRUE` is by far not as large as in case of `cv_method = "LOO"`.], `cv_varsel()` with `validate_search = FALSE` has comparable runtime to `varsel()`, but accounts for some overfitting, namely that induced by `varsel()`'s in-sample predictions during the predictive performance evaluation. However, as explained in section ["Variable selection"](#variableselection) (see also section ["Overfitting"](#overfitting)), `cv_varsel()` with `validate_search = FALSE` is more prone to overfitting than `cv_varsel()` with `validate_search = TRUE`. - For Gaussian models and the latent projection, `validate_search = TRUE` is usually fast enough (see above). + For the models with the fast computations above, `validate_search = TRUE` is usually fast enough. 1. Using `cv_varsel()` with subsampled PSIS-LOO CV [@magnusson_leave-one-out_2020], see argument `nloo` of `cv_varsel()`. @@ -554,8 +565,9 @@ Some speed-up possibilities are: 1. Using `varsel.vsel()` or `cv_varsel.vsel()` to re-use previous search results for new performance evaluation(s). This is helpful if the performance evaluation part is run multiple times based on the same search results (e.g., when only arguments `ndraws_pred` or `nclusters_pred` of `varsel()` or `cv_varsel()` are changed). - For example, `cv_varsel(cvvs, nclusters_pred = 20)` would re-use the searches of the `cvvs` object from above and run only a new performance evaluation. - Thus, a `cv_varsel()` call with `refit_prj = FALSE` (which has almost only the computational cost of the search) followed by `cv_varsel.vsel()` calls (which have only the computational cost of the performance evaluation) runs search and performance evaluation separately. + For example, `cv_varsel(cvvs, ndraws_pred = 200)` would re-use the searches of the `cvvs` object from above and run only a new performance evaluation. + Similarly, a `cv_varsel()` call with `refit_prj = FALSE` has almost only the computational cost of the search, and subsequent `cv_varsel.vsel()` calls have only the computational cost of the performance evaluation. + Note that the fold-wise searches of a `cv_varsel()` call with `refit_prj = FALSE` do not use the random observation weights (see argument `loo_perturb_alpha` of `cv_varsel()`). 1. Using `run_cvfun()` in case of repeated $K$-fold CV with the same $K$ reference model refits. The output of `run_cvfun()` is typically used as input for argument `cvfits` of `cv_varsel.refmodel()` (so in order to have a speed improvement, the output of `run_cvfun()` needs to be assigned to an object which is then re-used in multiple `cv_varsel()` calls). From 9ae6a6f60afb30291adc6eaa97b4d5f4c502a1e5 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 14:28:56 +0200 Subject: [PATCH 06/40] Use forward search and LOO CV with validated search in the latent-projection vignette --- vignettes/latent.Rmd | 145 +++++++++++++++++++++++++------------------ 1 file changed, 83 insertions(+), 62 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index ce15e832b..51a6296ce 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -62,7 +62,8 @@ The families currently supported by **projpred**'s augmented-data projection are ## Example: Poisson distribution -In this example, we will illustrate that in case of a family supported by **projpred**'s traditional projection (here the `poisson()` family), the latent projection can improve runtime and results of the variable selection compared to **projpred**'s traditional projection, at least if the L1 search is used (see argument `method` of `varsel()` and `cv_varsel()`). +In this example, we will illustrate that in case of a family supported by **projpred**'s traditional projection (here the `poisson()` family), the latent projection can improve the runtime of the variable selection compared to **projpred**'s traditional projection. +With L1 search (see argument `method` of `varsel()` and `cv_varsel()`), it can also improve the results. ### Data @@ -265,56 +266,85 @@ print(time_trad) As these results show, the traditional projection takes longer than the latent projection, although the difference is rather small on absolute scale (which is due to the fact that the L1 search is already quite fast). More importantly however, the predictor ranking contains several noise terms before truly relevant ones, causing the predictive performance of the reference model not to be reached before submodel size 28. +### Forward search + +Forward search is usually more accurate than L1 search (see section "Details" in `?varsel`). +We now run `varsel()` with forward search (the default `method`) for both projections: +```{r suppress_warn_poiss_fw, include=FALSE} +warn_instable_orig <- options(projpred.warn_instable_projections = FALSE) +``` +```{r vs_fw_poiss} +time_lat_fw <- system.time(vs_lat_fw <- varsel( + refm_poiss, + d_test = d_test_lat_poiss, + ### Only for the sake of speed (not recommended in general): + nclusters_pred = 20, + ### + nterms_max = 14, + verbose = 0, + seed = 95930 +)) +time_trad_fw <- system.time(vs_trad_fw <- varsel( + refm_fit_poiss, + d_test = d_test_trad_poiss, + ### Only for the sake of speed (not recommended in general): + nclusters_pred = 20, + ### + nterms_max = 14, + verbose = 0, + seed = 95930 +)) +rbind(latent = time_lat_fw, traditional = time_trad_fw)[, "elapsed"] +ranking(vs_lat_fw)[["fulldata"]] +ranking(vs_trad_fw)[["fulldata"]] +``` +```{r unsuppress_warn_poiss_fw, include=FALSE} +options(warn_instable_orig) +rm(warn_instable_orig) +``` +With forward search, both projections rank the 11 truly relevant predictor terms first (in a slightly different order). +The latent projection is much faster (see the elapsed times in seconds above). + +### LOO CV with validated search + +Without a hold-out test dataset, we would use `cv_varsel()` with PSIS-LOO CV and validated search (the defaults of `cv_varsel()`). +For the latent projection, this runs fast: +```{r cvvs_lat_poiss} +time_cvvs_lat <- system.time(cvvs_lat_poiss <- cv_varsel( + refm_poiss, + nterms_max = 14, + verbose = 0, + seed = 95930 +)) +print(time_cvvs_lat) +``` +```{r plot_cvvs_lat_poiss, fig.alt="The same kind of plot for LOO CV with validated search and the latent projection of the Poisson model. The ratio is close to 0 below size 9 and reaches the reference model at size 11."} +plot(cvvs_lat_poiss, stats = "gmpd", deltas = TRUE) +``` +```{r rk_cvvs_lat_poiss} +suggest_size(cvvs_lat_poiss, stat = "gmpd") +head(ranking(cvvs_lat_poiss)[["fulldata"]], 11) +``` +`suggest_size()` gives size 11, and the first 11 predictor terms of the full-data predictor ranking are the truly relevant ones. + +For the traditional projection, each projection onto a Poisson submodel is an iterative fit, so the same call takes much longer. +In a test run, it took about 390 seconds, compared to about 4 seconds for the latent projection above, and it ranked the same 11 truly relevant predictor terms first. +Therefore, we do not run it here: +```{r cvvs_trad_poiss, eval=FALSE} +cvvs_trad_poiss <- cv_varsel( + refm_fit_poiss, + nterms_max = 14, + verbose = 0, + seed = 95930 +) +``` + ### Conclusion -This example showed that the latent projection can be advantageous also for families supported by **projpred**'s traditional projection by improving the runtime as well as the results of the variable selection. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +This example showed that the latent projection can be advantageous also for families supported by **projpred**'s traditional projection. +With L1 search, the latent projection gave a better predictor ranking. +With forward search, both projections gave the same relevant predictor terms, but the latent projection was much faster, in particular in LOO CV with validated search. + An important point is that we have used L1 search here. In case of the latent projection, a forward search would have given only slightly different results. @@ -388,7 +418,6 @@ vs_nebin <- varsel( refm_nebin, d_test = d_test_lat_poiss, ### Only for the sake of speed (not recommended in general): - method = "L1", nclusters_pred = 20, ### nterms_max = 14, @@ -401,7 +430,7 @@ Again, the message telling that `$dis` consists of only `NA`s will not The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size and reaches the reference model at size 10 or 11. Up to size 11, the predictor ranking includes the noise term xn.29 and does not include x.9."} +```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, reaches the reference model at size 9 or 10 and levels off from size 10 on. The first 11 predictors of the ranking are the 11 truly relevant ones."} ( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) ``` @@ -415,7 +444,7 @@ This is not the size that `suggest_size()` would suggest, but as mentioned in th suggest_size(vs_nebin, stat = "gmpd") ``` -As we can see from the predictor ranking included in the plot, our selected `r if (!params$EVAL) character() else size_decided_nebin` predictor terms lack one truly relevant predictor (`x.9`) and include one noise term (`xn.29`). +As we can see from the predictor ranking included in the plot, our selected `r if (!params$EVAL) character() else size_decided_nebin` predictor terms are exactly the truly relevant ones. More explicitly, our selected predictor terms are: ```{r predictors_final_nebin} rk_nebin <- ranking(vs_nebin) @@ -567,10 +596,7 @@ if (use_parallel) { cvvs_weib <- cv_varsel( refm_weib, ### Only for the sake of speed (not recommended in general): - method = "L1", - nloo = min(N_surv, 10), nterms_max = 11, - nclusters_pred = 20, ### parallel = use_parallel, ### In interactive use, we recommend not to deactivate the verbose mode: @@ -583,10 +609,9 @@ if (use_parallel) { } ``` In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). -We will also ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. Plot the results: -```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model, with uncertainty bars, for the Weibull model with right-censored observations. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 7 on."} +```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model, with uncertainty bars, for the Weibull model with right-censored observations. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 9 on."} plot(cvvs_weib, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. @@ -709,10 +734,7 @@ if (use_parallel) { cvvs_lnorm <- cv_varsel( refm_lnorm, ### Only for the sake of speed (not recommended in general): - method = "L1", - nloo = min(N_surv, 10), nterms_max = 11, - nclusters_pred = 20, ### parallel = use_parallel, ### In interactive use, we recommend not to deactivate the verbose mode: @@ -725,10 +747,9 @@ if (use_parallel) { } ``` In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). -We will also ignore the warning that SIS is used (instead of PSIS) because this is due to `nclusters_pred = 20` which we used only to speed up the building of the vignette. Plot the results: -```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 8 on."} +```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) at size 10."} plot(cvvs_lnorm, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. From 3fe0272ace9039f8e18c9fc919d00842363bf281 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 14:29:17 +0200 Subject: [PATCH 07/40] Describe the Poisson example of the latent vignette with both search methods --- vignettes/projpred.Rmd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 2ce162d24..d227fca88 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -507,7 +507,8 @@ According to the simulation-based case study from [#353](https://github.com/stan For multilevel Poisson models, the traditional projection may take very long, see [#353](https://github.com/stan-dev/projpred/pull/353). According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in that case. -Finally, as illustrated in the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution), the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that is also supported by the traditional projection, at least in case of the Poisson family and L1 search. +Finally, as illustrated in the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution), the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that is also supported by the traditional projection, at least in case of the Poisson family. +In that example, the latent projection gave a better predictor ranking with L1 search, and it was much faster with forward search. ### Issues with the augmented-data projection From 5575dae8faf7259161e7511e21f0fffd7f8d8419 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 14:41:13 +0200 Subject: [PATCH 08/40] Correct the latent vignette text after review --- vignettes/latent.Rmd | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 51a6296ce..c9d5b2317 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -62,8 +62,8 @@ The families currently supported by **projpred**'s augmented-data projection are ## Example: Poisson distribution -In this example, we will illustrate that in case of a family supported by **projpred**'s traditional projection (here the `poisson()` family), the latent projection can improve the runtime of the variable selection compared to **projpred**'s traditional projection. -With L1 search (see argument `method` of `varsel()` and `cv_varsel()`), it can also improve the results. +In this example, we will illustrate that in case of a family supported by **projpred**'s traditional projection (here the `poisson()` family), the latent projection can reduce the runtime of the variable selection compared to the traditional projection. +With L1 search (see argument `method` of `varsel()` and `cv_varsel()`), the latent projection can also give a better predictor ranking. ### Data @@ -268,11 +268,8 @@ More importantly however, the predictor ranking contains several noise terms bef ### Forward search -Forward search is usually more accurate than L1 search (see section "Details" in `?varsel`). +Forward search can be more accurate than L1 search (see section "Details" in `?varsel`). We now run `varsel()` with forward search (the default `method`) for both projections: -```{r suppress_warn_poiss_fw, include=FALSE} -warn_instable_orig <- options(projpred.warn_instable_projections = FALSE) -``` ```{r vs_fw_poiss} time_lat_fw <- system.time(vs_lat_fw <- varsel( refm_poiss, @@ -281,7 +278,9 @@ time_lat_fw <- system.time(vs_lat_fw <- varsel( nclusters_pred = 20, ### nterms_max = 14, + ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0, + ### seed = 95930 )) time_trad_fw <- system.time(vs_trad_fw <- varsel( @@ -291,29 +290,29 @@ time_trad_fw <- system.time(vs_trad_fw <- varsel( nclusters_pred = 20, ### nterms_max = 14, + ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0, + ### seed = 95930 )) rbind(latent = time_lat_fw, traditional = time_trad_fw)[, "elapsed"] ranking(vs_lat_fw)[["fulldata"]] ranking(vs_trad_fw)[["fulldata"]] ``` -```{r unsuppress_warn_poiss_fw, include=FALSE} -options(warn_instable_orig) -rm(warn_instable_orig) -``` With forward search, both projections rank the 11 truly relevant predictor terms first (in a slightly different order). The latent projection is much faster (see the elapsed times in seconds above). ### LOO CV with validated search -Without a hold-out test dataset, we would use `cv_varsel()` with PSIS-LOO CV and validated search (the defaults of `cv_varsel()`). +Without a hold-out test dataset, we use `cv_varsel()` with PSIS-LOO CV and validated search (the defaults of `cv_varsel()`). For the latent projection, this runs fast: ```{r cvvs_lat_poiss} time_cvvs_lat <- system.time(cvvs_lat_poiss <- cv_varsel( refm_poiss, nterms_max = 14, + ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0, + ### seed = 95930 )) print(time_cvvs_lat) @@ -326,15 +325,20 @@ suggest_size(cvvs_lat_poiss, stat = "gmpd") head(ranking(cvvs_lat_poiss)[["fulldata"]], 11) ``` `suggest_size()` gives size 11, and the first 11 predictor terms of the full-data predictor ranking are the truly relevant ones. +The warnings about high Pareto $\hat{k}$ values (19 of the 71 observations) mean that PSIS-LOO CV may not be reliable here, probably because of the small number of observations and the short MCMC run of this vignette. +In a real analysis, we would then use $K$-fold CV (`cv_method = "kfold"`). -For the traditional projection, each projection onto a Poisson submodel is an iterative fit, so the same call takes much longer. -In a test run, it took about 390 seconds, compared to about 4 seconds for the latent projection above, and it ranked the same 11 truly relevant predictor terms first. +For the traditional projection, each projection onto a Poisson submodel is an iterative fit, and validated search repeats the search for each of the 71 LOO CV folds, so the same call takes much longer. +In a test run, the call took about 100 times as long as the latent projection (about 390 seconds compared to about 4 seconds). +It ranked the same 11 truly relevant predictor terms first, and `suggest_size()` gave size 12. Therefore, we do not run it here: ```{r cvvs_trad_poiss, eval=FALSE} cvvs_trad_poiss <- cv_varsel( refm_fit_poiss, nterms_max = 14, + ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0, + ### seed = 95930 ) ``` @@ -343,13 +347,8 @@ cvvs_trad_poiss <- cv_varsel( This example showed that the latent projection can be advantageous also for families supported by **projpred**'s traditional projection. With L1 search, the latent projection gave a better predictor ranking. -With forward search, both projections gave the same relevant predictor terms, but the latent projection was much faster, in particular in LOO CV with validated search. - - -An important point is that we have used L1 search here. -In case of the latent projection, a forward search would have given only slightly different results. -However, in case of the traditional projection, a forward search would have given markedly better results (in particular, all of the noise terms would have been selected after the truly relevant ones). -Thus, the conclusions made here for L1 search cannot be transmitted easily to forward search. +With forward search, both projections ranked the 11 truly relevant predictor terms first. +The latent projection was much faster, in particular in LOO CV with validated search. ## Example: Negative binomial distribution {#negbinex} @@ -430,7 +429,7 @@ Again, the message telling that `$dis` consists of only `NA`s will not The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, reaches the reference model at size 9 or 10 and levels off from size 10 on. The first 11 predictors of the ranking are the 11 truly relevant ones."} +```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, is above 1 from size 9 on and levels off at about 1.2 from size 10 on. The first 11 predictors of the ranking are the 11 truly relevant ones."} ( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) ``` @@ -749,7 +748,7 @@ if (use_parallel) { In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). Plot the results: -```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) at size 10."} +```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 10 on."} plot(cvvs_lnorm, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. From 9a46d7bf0fdfd9da84a61bed45eee870f94ec8e0 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 14:41:35 +0200 Subject: [PATCH 09/40] Shorten the Troubleshooting sentence on the latent Poisson example --- vignettes/projpred.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index d227fca88..af2f40c41 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -507,7 +507,7 @@ According to the simulation-based case study from [#353](https://github.com/stan For multilevel Poisson models, the traditional projection may take very long, see [#353](https://github.com/stan-dev/projpred/pull/353). According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in that case. -Finally, as illustrated in the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution), the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that is also supported by the traditional projection, at least in case of the Poisson family. +Finally, the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution) shows that the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that the traditional projection also supports. In that example, the latent projection gave a better predictor ranking with L1 search, and it was much faster with forward search. ### Issues with the augmented-data projection From a50761845dafe1b2d876e3018a1815806173afd2 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 19:28:29 +0200 Subject: [PATCH 10/40] Edit projpred vignette --- vignettes/projpred.Rmd | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index af2f40c41..d4228666a 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -100,6 +100,8 @@ refm_fit <- stan_glm( prior = hs(global_scale = tau0), ### Only for the sake of speed (not recommended in general): chains = 2, iter = 1000, + ### In general use defaults or explicitly define + ### chains = 4, iter = 4000, ### refresh = 0 ) @@ -134,9 +136,7 @@ It can also help to choose argument `nterms_max` (the largest submodel size of t Here, we run `cv_varsel()` with its defaults: PSIS-LOO CV (`cv_method = "LOO"`), validated search (`validate_search = TRUE`) and forward search (`method = "forward"`). For a Gaussian reference model with the identity link and without multilevel or additive terms, like this one, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix. -Therefore, this run takes only a few seconds. -In LOO CV with validated search, the search of each fold uses random observation weights (see argument `loo_perturb_alpha` of `cv_varsel()`). -This makes the estimated predictive performance change more smoothly between submodel sizes. +This run takes only a few seconds. The code below also shows how **projpred**'s CV can be parallelized with the **future** package (we disable the parallelization for the vignette build). The LOO folds are then split among the parallel workers. @@ -166,7 +166,7 @@ if (use_parallel) { #### Predictive performance plot from the `cv_varsel()` run {#plotfinal} -We can now select a final submodel size by looking at a predictive performance plot. +We can now select a submodel size by looking at a predictive performance plot. By default, the performance statistics are plotted on their actual scale and the uncertainty bars match this scale, but argument `deltas` of `plot.vsel()` offers two more options: * With `deltas = TRUE`, the performance statistics are plotted on *difference scale*, i.e., as differences^[For the geometric mean predictive density (GMPD; see argument `stats` of `summary.vsel()` and `plot.vsel()`), `deltas = TRUE` estimates the GMPD *ratio* (not difference) vs. the baseline model.] from the baseline model^[For the definition of the baseline model, see argument `baseline` of `summary.vsel()` and `plot.vsel()`; in the most common cases, the default baseline model is the reference model.] and the uncertainty bars match this scale, @@ -244,10 +244,10 @@ To compute these ranking proportions, we use `cv_proportions()`: ``` The main diagonal of this matrix is contained in column `cv_proportions_diag` of the `summary.vsel()` output and is also shown in the third line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from the `cv_varsel()` run"](#plotfinal) (due to `show_cv_proportions = TRUE` set via its global option). -Here, 93 % of the LOO CV folds have `X1` at position 1 and `X14` at position 2. +Here, 93% of the LOO CV folds have `X1` at position 1 and `X14` at position 2. The other folds have these two predictors in the opposite order. -At each of positions 3 to 8, at least 96 % of the folds have the same predictor as the full-data predictor ranking at that position. -From position 9 on, the proportions are below 40 %, so the ranking of the remaining predictors varies between the folds. +At each of positions 3 to 8, at least 96% of the folds have the same predictor as the full-data predictor ranking at that position. +From position 9 on, the proportions are below 40%, so the ranking of the remaining predictors varies between the folds. Since the column names of the matrix returned by `cv_proportions()` follow the full-data predictor ranking, we can infer that `X1` and `X14` are also the most relevant predictor terms (in this order) in the full-data predictor ranking. To see this more explicitly, we can access element `fulldata` of the `ranking()` output: ```{r ranking_fulldata} @@ -256,7 +256,7 @@ rk[["fulldata"]] This is the same as column `ranking_fulldata` in the `summary.vsel()` output above (apart from the intercept). We cut off the predictor rankings at 12 predictor terms, which is smaller than the number of predictor terms in the full model (20 here). -Therefore, the ranking proportions in the `pr_rk` matrix do not need to sum to 100 % (neither column-wise nor row-wise). +Therefore, the ranking proportions in the `pr_rk` matrix do not need to sum to 100% (neither column-wise nor row-wise). The *transposed* matrix of ranking proportions can be visualized via `plot.cv_proportions()`: ```{r plot_cv_proportions, fig.alt="Heat map of the ranking proportions. The y-axis shows the predictors in the order of the full-data predictor ranking and the x-axis the ranking positions 1 to 12. Each tile gives the proportion of CV folds that have the predictor at that position."} @@ -274,13 +274,13 @@ plot(cv_proportions(rk, cumulate = TRUE)) ``` This plot shows that all fold-wise searches and the full-data search agree on the *set* of the 7 most relevant predictors. (The full-data predictor ranking determines the order of the predictors on the y-axis.) -At `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100 %. +At `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100%. The same holds for the set of the 8 most relevant predictors (`<=8`), but not for larger sets. -For example, only 35 % of the folds have `X10` among their first 9 predictors. +For example, only 35% of the folds have `X10` among their first 9 predictors. The cumulated ranking proportions can also guide the decision for a submodel size. Here, they agree with our choice of size 7: all folds select the same 7 (and 8) predictors. -A size of 9 would add `X10`, which only 35 % of the folds have among their first 9 predictors. +A size of 9 would add `X10`, which only 35% of the folds have among their first 9 predictors. In a real-world application, we might also be able to incorporate the full-data predictor ranking into our decision for a submodel size (usually, this requires to also take into account the variability of the predictor ranking, as reflected by the---possibly cumulated---ranking proportions). For example, the predictors might be associated with different measurement costs, so that we might want to select a costly predictor only if the submodel size at which it would be selected (according to the full-data predictor ranking, but taking into account that there might be variability in the ranking of the predictors) comes with a considerable increase in predictive performance. @@ -349,7 +349,7 @@ A visualization of the projected posterior can be achieved with the [**bayesplot library(bayesplot) ``` ```{r bayesplot_prj, fig.alt="Interval plot of the projected posterior, with one row for the intercept, each of the 7 selected predictors and sigma, and the parameter value on the x-axis. Each row has a point for the median, a thick line for the 50 % interval and a thin line for the 90 % interval."} -bayesplot_theme_set(ggplot2::theme_bw()) +ggplot2::theme_set(bayesplot::theme_default(base_family = "sans")) mcmc_intervals(prj_mat) + ggplot2::coord_cartesian(xlim = c(-1.5, 1.6)) ``` @@ -508,7 +508,6 @@ For multilevel Poisson models, the traditional projection may take very long, se According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in that case. Finally, the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution) shows that the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that the traditional projection also supports. -In that example, the latent projection gave a better predictor ranking with L1 search, and it was much faster with forward search. ### Issues with the augmented-data projection @@ -519,7 +518,7 @@ According to the simulation-based case study from [#353](https://github.com/stan ### Speed {#speed} For Gaussian models with the identity link and without multilevel or additive terms, and for the latent projection without such terms, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix. -Therefore, the defaults of `cv_varsel()` are usually fast for these models. +The defaults of `cv_varsel()` are usually fast for these models. For other models, each projection is an iterative fit, and `cv_varsel()` with the defaults can take much longer. For binomial and Poisson models, the latent projection is an alternative that uses the same fast computations (see the [latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html)). From 901d8512d888a4580f70588d0bdf0a05f3369a7b Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 20:13:03 +0200 Subject: [PATCH 11/40] Use the formula y ~ . for the reference model of the main vignette --- vignettes/projpred.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index d4228666a..caf249b29 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -60,6 +60,7 @@ library(rstanarm) For our **rstanarm** reference model, we use the Gaussian distribution as the `family` for our response. With respect to the predictors, we only include the linear main effects of all 20 predictor variables. +The formula `y ~ .` uses all other columns of the data as predictors. Compared to the more complex types of reference models supported by **projpred** (see section ["Supported types of models"](#modtypes) below), this is a quite simple reference model which is sufficient, however, to demonstrate the interplay of **projpred**'s functions. We use **rstanarm**'s default priors in our reference model, except for the regression coefficients for which we use a regularized horseshoe prior [@piironen_sparsity_2017] with the hyperprior for its global shrinkage parameter following @piironen_hyperprior_2017 and @piironen_sparsity_2017. @@ -93,8 +94,7 @@ ncores <- min(ncores, 2L) options(mc.cores = ncores) set.seed(50780) refm_fit <- stan_glm( - y ~ X1 + X2 + X3 + X4 + X5 + X6 + X7 + X8 + X9 + X10 + X11 + X12 + X13 + X14 + - X15 + X16 + X17 + X18 + X19 + X20, + y ~ ., family = gaussian(), data = dat_gauss, prior = hs(global_scale = tau0), From 53293ecdad7e59adb703fb7cce2dc172e106ffbc Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 20:39:07 +0200 Subject: [PATCH 12/40] Explain the submodels above the reference model, and use y ~ . in the latent vignette --- vignettes/latent.Rmd | 6 ++---- vignettes/projpred.Rmd | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index c9d5b2317..6ae7cbf2d 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -137,10 +137,8 @@ ncores <- parallel::detectCores(logical = FALSE) ncores <- min(ncores, 2L) ### options(mc.cores = ncores) -refm_fml <- as.formula(paste("y", "~", paste( - grep("^x", names(dat_poiss_train), value = TRUE), - collapse = " + " -))) +# All other columns of the data are predictors: +refm_fml <- y ~ . refm_fit_poiss <- stan_glm( formula = refm_fml, family = poisson(), diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index caf249b29..b8d9481f4 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -191,7 +191,8 @@ Sometimes, the plot may be ambiguous because after reaching the reference model' In that case, one has to find a suitable trade-off between predictive performance (accuracy) and model size (sparsity) in the context of subject-matter knowledge. Here, we decide for a submodel size of 7 because it seems to provide the best trade-off between sparsity and accuracy (size 7 is the smallest size where the submodel MLPD is close enough to the reference model MLPD and from size 7 on, the submodel MLPD levels off). -At sizes 7 to 10, the submodel MLPD is even slightly higher than the reference model MLPD, which is the case of submodels that perform better than the reference model mentioned above. +At sizes 7 to 10, the submodels predict slightly better than the reference model. +The reference model also includes 12 predictors with little or no effect, and their posterior uncertainty adds predictive variance, which the projection onto the smaller submodels removes. ```{r size_man} size_decided <- 7 ``` From 6fb251be6f55f31c723ec0d0b25bb452f78138c0 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 23:04:15 +0200 Subject: [PATCH 13/40] Use the workers of future.mirai in the parallel examples of the latent vignette --- vignettes/latent.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 6ae7cbf2d..d5b81e09d 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -588,7 +588,7 @@ Run `cv_varsel()`: # Note: Parallel processing is disabled during package building to avoid issues use_parallel <- FALSE # Set to TRUE for actual parallel processing if (use_parallel) { - future::plan(future::multisession, workers = ncores) + future::plan(future.mirai::mirai_multisession, workers = ncores) } cvvs_weib <- cv_varsel( refm_weib, @@ -726,7 +726,7 @@ Run `cv_varsel()`: # Note: Parallel processing is disabled during package building to avoid issues use_parallel <- FALSE # Set to TRUE for actual parallel processing if (use_parallel) { - future::plan(future::multisession, workers = ncores) + future::plan(future.mirai::mirai_multisession, workers = ncores) } cvvs_lnorm <- cv_varsel( refm_lnorm, From fb7c1cef38de7f9028682062b76199ed3b589e92 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Mon, 16 Nov 2026 23:20:18 +0200 Subject: [PATCH 14/40] Use the workers of future.mirai in the parallel examples of the main vignette --- vignettes/projpred.Rmd | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index b8d9481f4..34799a60f 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -100,7 +100,7 @@ refm_fit <- stan_glm( prior = hs(global_scale = tau0), ### Only for the sake of speed (not recommended in general): chains = 2, iter = 1000, - ### In general use defaults or explicitly define + ### In general, use defaults or explicitly define ### chains = 4, iter = 4000, ### refresh = 0 @@ -141,6 +141,7 @@ This run takes only a few seconds. The code below also shows how **projpred**'s CV can be parallelized with the **future** package (we disable the parallelization for the vignette build). The LOO folds are then split among the parallel workers. For a model as small as this one, the parallelization overhead can be larger than the runtime gain. +We recommend the workers of the **future.mirai** package: in our experiments, the first parallel call was faster with them than with `future::multisession` (see section "Parallelization" in the general package documentation available [online](https://mc-stan.org/projpred/reference/projpred-package.html) or by typing `` ?`projpred-package` ``). If PSIS-LOO CV is not reliable (for example, if many Pareto $\hat{k}$ values are high), $K$-fold CV (`cv_method = "kfold"`) is an alternative. Then `run_cvfun()` can run the $K$ reference model refits once, so that several `cv_varsel()` calls can use them (see argument `cvfits` of `cv_varsel()`). @@ -149,7 +150,7 @@ Then `run_cvfun()` can run the $K$ reference model refits once, so that several # Note: Parallel processing is disabled during package building to avoid issues use_parallel <- FALSE # Set to TRUE for actual parallel processing if (use_parallel) { - future::plan(future::multisession, workers = ncores) + future::plan(future.mirai::mirai_multisession, workers = ncores) } cvvs <- cv_varsel( refm_obj, @@ -298,7 +299,7 @@ By the following `project()` call, we project the reference model onto the final - + @@ -561,7 +562,7 @@ Some speed-up possibilities are: In case of forward search, this issue does not exist. 1. Parallelizing the CV folds of `cv_varsel()` with the **future** package (see the example above and argument `parallel` of `cv_varsel()`). - The parallel workers are set up with `future::plan()`, for example `future::plan(future::multisession, workers = 4)`. + The parallel workers are set up with `future::plan()`, for example `future::plan(future.mirai::mirai_multisession, workers = 4)` (recommended) or `future::plan(future::multisession, workers = 4)`. When using `project()`, parallelizing the projection might also help (see the general package documentation available [online](https://mc-stan.org/projpred/reference/projpred-package.html) or by typing `` ?`projpred-package` ``). 1. Using `varsel.vsel()` or `cv_varsel.vsel()` to re-use previous search results for new performance evaluation(s). From 289b5a2b3136863a665140754c573598b0b6c886 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 09:25:31 +0200 Subject: [PATCH 15/40] Describe the ranking proportions without exact percentages --- vignettes/projpred.Rmd | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 34799a60f..a6f4625df 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -246,10 +246,9 @@ To compute these ranking proportions, we use `cv_proportions()`: ``` The main diagonal of this matrix is contained in column `cv_proportions_diag` of the `summary.vsel()` output and is also shown in the third line of the x-axis tick labels of the predictive performance plot from section ["Predictive performance plot from the `cv_varsel()` run"](#plotfinal) (due to `show_cv_proportions = TRUE` set via its global option). -Here, 93% of the LOO CV folds have `X1` at position 1 and `X14` at position 2. -The other folds have these two predictors in the opposite order. -At each of positions 3 to 8, at least 96% of the folds have the same predictor as the full-data predictor ranking at that position. -From position 9 on, the proportions are below 40%, so the ranking of the remaining predictors varies between the folds. +Here, most LOO CV folds have `X1` at position 1 and `X14` at position 2. +At positions 3 to 8, most folds also have the same predictor as the full-data predictor ranking. +From position 9 on, the proportions are smaller, so the ranking of the remaining predictors varies between the folds. Since the column names of the matrix returned by `cv_proportions()` follow the full-data predictor ranking, we can infer that `X1` and `X14` are also the most relevant predictor terms (in this order) in the full-data predictor ranking. To see this more explicitly, we can access element `fulldata` of the `ranking()` output: ```{r ranking_fulldata} @@ -271,18 +270,18 @@ To retrieve the predictor terms of the final submodel (except for the intercept ( predictors_final <- head(rk[["fulldata"]], size_decided) ) ``` At this place, it is again helpful to take the ranking proportions into account, but now in a cumulated fashion: -```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 12 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At sizes 7 and 8, all tiles of the first 7 or 8 predictors are at 100 %, at size 9 only 35 % of the folds include the 9th predictor."} +```{r plot_cv_proportions_cumul, fig.alt="Heat map of the cumulated ranking proportions. For each predictor (y-axis) and each size j from 1 to 12 (x-axis), the tile gives the proportion of CV folds that have the predictor among their first j predictors. At sizes 7 and 8, all tiles of the first 7 or 8 predictors are at 100 %; at size 9, fewer folds include the 9th predictor."} plot(cv_proportions(rk, cumulate = TRUE)) ``` This plot shows that all fold-wise searches and the full-data search agree on the *set* of the 7 most relevant predictors. (The full-data predictor ranking determines the order of the predictors on the y-axis.) At `<=7` on the x-axis, all tiles above and including the 7^th^ main diagonal element are at 100%. The same holds for the set of the 8 most relevant predictors (`<=8`), but not for larger sets. -For example, only 35% of the folds have `X10` among their first 9 predictors. +For example, only some of the folds have `X10` among their first 9 predictors. The cumulated ranking proportions can also guide the decision for a submodel size. Here, they agree with our choice of size 7: all folds select the same 7 (and 8) predictors. -A size of 9 would add `X10`, which only 35% of the folds have among their first 9 predictors. +A size of 9 would add `X10`, which only some of the folds have among their first 9 predictors. In a real-world application, we might also be able to incorporate the full-data predictor ranking into our decision for a submodel size (usually, this requires to also take into account the variability of the predictor ranking, as reflected by the---possibly cumulated---ranking proportions). For example, the predictors might be associated with different measurement costs, so that we might want to select a costly predictor only if the submodel size at which it would be selected (according to the full-data predictor ranking, but taking into account that there might be variability in the ranking of the predictors) comes with a considerable increase in predictive performance. From 3b303dd058c7aa189ef1f06b1674131abeae8beb Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 09:26:48 +0200 Subject: [PATCH 16/40] Describe the plots of the latent vignette in the alt texts without exact sizes and ratios --- vignettes/latent.Rmd | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index d5b81e09d..e477afed6 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -200,7 +200,7 @@ In order to decide for a submodel size, we first inspect the `plot()` results. In contrast to the main vignette where we used the mean log predictive density (MLPD) as predictive performance statistic for a `gaussian()` family reference model (and `gaussian()` submodels), we have a discrete family (`poisson()`) here, so it makes sense to exponentiate the MLPD to obtain the geometric mean predictive density (GMPD; in case of a discrete response, the predictive density values are actually predictive *probabilities* and hence the GMPD is bounded by 0 and 1). As in the main vignette, we plot with `deltas = TRUE` (in case of the GMPD, this means that the *ratio* of the submodel GMPD vs. the reference model GMPD is shown). Via global option `projpred.plot_vsel_size_position`, we set argument `size_position` of `plot.vsel()` to `"secondary_x"` to make the submodel sizes readable in all of the plots in this vignette. -```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the latent projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 below size 11 and reaches the reference model at size 11."} +```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the latent projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 at small sizes and then rises to the reference model."} options(projpred.plot_vsel_size_position = "secondary_x") ( gg_lat <- plot(vs_lat, stats = "gmpd", deltas = TRUE) ) ``` @@ -257,7 +257,7 @@ time_trad <- system.time(vs_trad <- varsel( options(warn_instable_orig) rm(warn_instable_orig) ``` -```{r post_vs_trad, fig.alt="The same plot for the traditional projection. Several noise terms enter the predictor ranking before relevant ones, and the submodels reach the reference model only at size 28."} +```{r post_vs_trad, fig.alt="The same plot for the traditional projection. Several noise terms enter the predictor ranking before relevant ones, and the submodels reach the reference model only at a much larger size."} print(time_trad) ( gg_trad <- plot(vs_trad, stats = "gmpd", deltas = TRUE) ) ``` @@ -315,7 +315,7 @@ time_cvvs_lat <- system.time(cvvs_lat_poiss <- cv_varsel( )) print(time_cvvs_lat) ``` -```{r plot_cvvs_lat_poiss, fig.alt="The same kind of plot for LOO CV with validated search and the latent projection of the Poisson model. The ratio is close to 0 below size 9 and reaches the reference model at size 11."} +```{r plot_cvvs_lat_poiss, fig.alt="The same kind of plot for LOO CV with validated search and the latent projection of the Poisson model. The ratio is close to 0 at small sizes and then rises to the reference model."} plot(cvvs_lat_poiss, stats = "gmpd", deltas = TRUE) ``` ```{r rk_cvvs_lat_poiss} @@ -427,7 +427,7 @@ Again, the message telling that `$dis` consists of only `NA`s will not The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, is above 1 from size 9 on and levels off at about 1.2 from size 10 on. The first 11 predictors of the ranking are the 11 truly relevant ones."} +```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, rises above 1 and then levels off. The first 11 predictors of the ranking are the 11 truly relevant ones."} ( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) ``` @@ -608,7 +608,7 @@ if (use_parallel) { In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). Plot the results: -```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model, with uncertainty bars, for the Weibull model with right-censored observations. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 9 on."} +```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model, with uncertainty bars, for the Weibull model with right-censored observations. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. The first 10 predictors are the 10 truly relevant ones. The difference increases with the size and then levels off close to zero (dashed line)."} plot(cvvs_weib, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. @@ -746,7 +746,7 @@ if (use_parallel) { In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). Plot the results: -```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference comes close to zero (dashed line) from size 10 on."} +```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference increases with the size and then levels off close to zero (dashed line)."} plot(cvvs_lnorm, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. From d55ab8c55c2752a6b38876528c82c5d46118ed89 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 09:28:17 +0200 Subject: [PATCH 17/40] Remove the commented-out projection parallelization from the main vignette --- vignettes/projpred.Rmd | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index a6f4625df..45ab72e38 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -290,18 +290,6 @@ For example, the predictors might be associated with different measurement costs The `project()` function returns an object of class `projection` which forms the basis for convenient post-selection inference. By the following `project()` call, we project the reference model onto the final submodel once again^[During the search, the reference model is projected onto all candidate models (this is where arguments `ndraws` and `nclusters` of `varsel()` and `cv_varsel()` come into play; note that in case of an L1 search, the projection is L1-penalized). For the evaluation of the submodels along the predictor ranking returned by the search, the reference model is projected onto these submodels again (this is where arguments `ndraws_pred` and `nclusters_pred` of `varsel()` and `cv_varsel()` come into play; note that this only holds if argument `refit_prj` of `varsel()` and `cv_varsel()` is set to `TRUE`, as by default). Within `project()`, `refit_prj = FALSE` allows to re-use the submodel fits (that is, the projections) from the full-data search of a `vsel` object, but usually, the search relies on a rather coarse clustering or thinning of the reference model's posterior draws (by default, `varsel()` and `cv_varsel()` use `nclusters = 20`---or `nclusters = 1` in case of L1 search), which would then imply the same coarseness for a `project()` call where `refit_prj` is set to `FALSE`. In general, we want the final projection (that post-selection inference is based on) to be as accurate as possible, so here we call `project()` with the defaults of `refit_prj = TRUE` and `ndraws = 400`. For more accurate results, we could increase argument `ndraws` of `project()` (up to the number of posterior draws in the reference model). However, this would increase the runtime, which we don't want in this vignette.]: - - - - - - - - - - - - ```{r project} prj <- project( refm_obj, @@ -311,12 +299,6 @@ prj <- project( ### ) ``` - - - - - - Next, we create a matrix containing the projected posterior draws stored in the depths of `project()`'s output: ```{r as_matrix_prj} @@ -430,9 +412,7 @@ On the side of the predictors, **projpred** not only supports linear main effect Transferring this vignette to such more complex problems is straightforward (also because this vignette employs a "typical" reference model object): Basically, only the code for fitting the reference model via **rstanarm** or **brms** needs to be adapted. The **projpred** code stays almost the same. -Only note that in case of multilevel or additive reference models, - -some **projpred** functions then have slightly different options for a few arguments. +Only note that in case of multilevel or additive reference models, some **projpred** functions then have slightly different options for a few arguments. See the documentation for details. For example, to apply **projpred** to the `VerbAgg` dataset from the [**lme4**](https://CRAN.R-project.org/package=lme4) package, a corresponding multilevel reference model for the binary response `r2` could be created by the following code: From 8a9d09daec52fb5a6d2b1b28e2281faca06b715c Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 09:31:44 +0200 Subject: [PATCH 18/40] Do not present L1 search as a reason for the latent projection --- vignettes/latent.Rmd | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index e477afed6..c1cc4d26c 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -63,7 +63,8 @@ The families currently supported by **projpred**'s augmented-data projection are ## Example: Poisson distribution In this example, we will illustrate that in case of a family supported by **projpred**'s traditional projection (here the `poisson()` family), the latent projection can reduce the runtime of the variable selection compared to the traditional projection. -With L1 search (see argument `method` of `varsel()` and `cv_varsel()`), the latent projection can also give a better predictor ranking. +We first use L1 search (see argument `method` of `varsel()` and `cv_varsel()`) only to show an issue of the traditional projection. +In general, forward search (the default) is more accurate than L1 search. ### Data @@ -266,7 +267,7 @@ More importantly however, the predictor ranking contains several noise terms bef ### Forward search -Forward search can be more accurate than L1 search (see section "Details" in `?varsel`). +In general, forward search is more accurate than L1 search (see section "Details" in `?varsel`). We now run `varsel()` with forward search (the default `method`) for both projections: ```{r vs_fw_poiss} time_lat_fw <- system.time(vs_lat_fw <- varsel( @@ -344,7 +345,7 @@ cvvs_trad_poiss <- cv_varsel( ### Conclusion This example showed that the latent projection can be advantageous also for families supported by **projpred**'s traditional projection. -With L1 search, the latent projection gave a better predictor ranking. +With L1 search, which we used only to show an issue of the traditional projection, the traditional projection ranked several noise terms before truly relevant ones. With forward search, both projections ranked the 11 truly relevant predictor terms first. The latent projection was much faster, in particular in LOO CV with validated search. From abd1f5c3690554e90e5e73015ab951f224d399ac Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:06:36 +0200 Subject: [PATCH 19/40] Remove the early message about an NA dispersion of a latent reference model --- R/refmodel.R | 9 --------- 1 file changed, 9 deletions(-) diff --git a/R/refmodel.R b/R/refmodel.R index d9f6b8652..5fceb1f52 100644 --- a/R/refmodel.R +++ b/R/refmodel.R @@ -1787,15 +1787,6 @@ init_refmodel <- function(object, data, formula, family, ref_predfun = NULL, } else { dis <- rep(NA, ndraws) } - if (all(is.na(dis))) { - message( - "Since `$dis` will consist of only `NA`s, downstream ", - "analyses based on this reference model object won't be able to use ", - "log predictive density (LPD) values on latent scale. Furthermore, ", - "proj_predict() won't be able to draw from the latent Gaussian ", - "distribution." - ) - } } else if (!has_dispersion(family)) { dis <- rep(NA, ndraws) } else { From 6dce828e581c4a64a517e2632b243d98ce4dee8c Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:07:52 +0200 Subject: [PATCH 20/40] Remove the vignette notes on the NA dispersion message --- vignettes/latent.Rmd | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index c1cc4d26c..146de9ceb 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -147,6 +147,8 @@ refm_fit_poiss <- stan_glm( prior = hs(global_scale = tau0, slab_df = 100, slab_scale = 1), ### Only for the sake of speed (not recommended in general): chains = 2, iter = 1000, + ### In general, use defaults or explicitly define + ### chains = 4, iter = 4000, ### refresh = 0 ) @@ -195,13 +197,13 @@ time_lat <- system.time(vs_lat <- varsel( ```{r time_lat} print(time_lat) ``` -The message telling that `$dis` consists of only `NA`s will not concern us here because we will only focus on response-scale post-processing. In order to decide for a submodel size, we first inspect the `plot()` results. In contrast to the main vignette where we used the mean log predictive density (MLPD) as predictive performance statistic for a `gaussian()` family reference model (and `gaussian()` submodels), we have a discrete family (`poisson()`) here, so it makes sense to exponentiate the MLPD to obtain the geometric mean predictive density (GMPD; in case of a discrete response, the predictive density values are actually predictive *probabilities* and hence the GMPD is bounded by 0 and 1). As in the main vignette, we plot with `deltas = TRUE` (in case of the GMPD, this means that the *ratio* of the submodel GMPD vs. the reference model GMPD is shown). Via global option `projpred.plot_vsel_size_position`, we set argument `size_position` of `plot.vsel()` to `"secondary_x"` to make the submodel sizes readable in all of the plots in this vignette. ```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the latent projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 at small sizes and then rises to the reference model."} +ggplot2::theme_set(bayesplot::theme_default(base_family = "sans")) options(projpred.plot_vsel_size_position = "secondary_x") ( gg_lat <- plot(vs_lat, stats = "gmpd", deltas = TRUE) ) ``` @@ -424,7 +426,6 @@ vs_nebin <- varsel( ### ) ``` -Again, the message telling that `$dis` consists of only `NA`s will not concern us here because we will only focus on response-scale post-processing. The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: @@ -577,7 +578,6 @@ refm_weib <- get_refmodel( latent_ppd_oscale = latent_ppd_oscale_weib ) ``` -Again, the message telling that `$dis` consists of only `NA`s will not concern us here because we will only focus on response-scale post-processing. The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). The functions above use the shape draws of the full-data fit (`refm_shape`). This is correct for `varsel()` and for the LOO CV that we use below. @@ -619,7 +619,6 @@ Project onto the submodel consisting of the first `n_pred_truth` predictors and predictors_final_weib <- head(ranking(cvvs_weib)[["fulldata"]], n_pred_truth) prj_weib <- project(refm_weib, predictor_terms = predictors_final_weib) prj_predict_weib <- proj_predict(prj_weib) -bayesplot::bayesplot_theme_set(ggplot2::theme_bw()) bayesplot::ppc_km_overlay(y = dat_sim_weib$yobs, yrep = prj_predict_weib, status_y = 1 - dat_sim_weib$is_censored) ``` @@ -825,7 +824,7 @@ refm_nebin_kfold <- get_refmodel_nebin( } ) ``` -As before, the messages concerning `$dis` and `latent_ilink` can be ignored here. +As before, the message concerning `latent_ilink` can be ignored here. Then `cv_varsel(refm_nebin_kfold, cv_method = "kfold")` uses the `reciprocal_dispersion` draws of the fold-wise refit in each fold. We do not run K-fold CV here, because it refits the reference model $K$ times. For the Weibull example, define `refm_shape` inside such a function in the same way. From 19ee476bc52e1bcceb40f8a8b696adb55ef3a9d7 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:08:12 +0200 Subject: [PATCH 21/40] Skip the latent_ilink message for families whose linkinv gives the mean --- R/extend_family.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/R/extend_family.R b/R/extend_family.R index 0bc24afe5..c85ccdcce 100644 --- a/R/extend_family.R +++ b/R/extend_family.R @@ -338,7 +338,11 @@ extend_family <- function(family, return(ilpreds) } } else { - if (family$family_oscale != "poisson") { + # Families whose `linkinv` gives the mean of the response: + fams_linkinv_mean <- c("poisson", "neg_binomial_2", "negbinomial", + "geometric", "Gamma", "weibull", + "exponential") + if (!family$family_oscale %in% fams_linkinv_mean) { message("Defining `latent_ilink` as a function which calls ", "`family$linkinv`, but there is no guarantee that this ", "will work for all families. If relying on ", From b56c658caa038ae4f9b99c4f79d38526c1f0343a Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:08:35 +0200 Subject: [PATCH 22/40] Remove the vignette notes on the latent_ilink message where it no longer appears --- vignettes/latent.Rmd | 3 --- 1 file changed, 3 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 146de9ceb..c360e160f 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -426,7 +426,6 @@ vs_nebin <- varsel( ### ) ``` -The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). Again, we first inspect the `plot()` results to decide for a submodel size: ```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, rises above 1 and then levels off. The first 11 predictors of the ranking are the 11 truly relevant ones."} @@ -578,7 +577,6 @@ refm_weib <- get_refmodel( latent_ppd_oscale = latent_ppd_oscale_weib ) ``` -The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). The functions above use the shape draws of the full-data fit (`refm_shape`). This is correct for `varsel()` and for the LOO CV that we use below. For K-fold CV, see section ["K-fold CV with custom response-scale functions"](#kfold-custom). @@ -824,7 +822,6 @@ refm_nebin_kfold <- get_refmodel_nebin( } ) ``` -As before, the message concerning `latent_ilink` can be ignored here. Then `cv_varsel(refm_nebin_kfold, cv_method = "kfold")` uses the `reciprocal_dispersion` draws of the fold-wise refit in each fold. We do not run K-fold CV here, because it refits the reference model $K$ times. For the Weibull example, define `refm_shape` inside such a function in the same way. From 2172423509b985e440b0e0c883d1ebf5d67e69d4 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:08:52 +0200 Subject: [PATCH 23/40] Add a NEWS item for the fewer messages of latent reference models --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index c239ff92b..4bdb69f37 100644 --- a/NEWS.md +++ b/NEWS.md @@ -44,6 +44,7 @@ If you read this from a place other than Date: Tue, 17 Nov 2026 10:14:06 +0200 Subject: [PATCH 24/40] Add more mean-parameterized families to the latent_ilink check, and test the latent reference model messages --- NEWS.md | 2 +- R/extend_family.R | 15 ++++++++++----- tests/testthat/test_extend_family.R | 27 +++++++++++++++++++++++++++ tests/testthat/test_latent.R | 15 +++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 4bdb69f37..105cb44c6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -44,7 +44,7 @@ If you read this from a place other than Date: Tue, 17 Nov 2026 10:39:44 +0200 Subject: [PATCH 25/40] Test the NA dispersion message also with the brms Poisson fit of the setup --- tests/testthat/test_latent.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test_latent.R b/tests/testthat/test_latent.R index 401881434..d5c41b2fd 100644 --- a/tests/testthat/test_latent.R +++ b/tests/testthat/test_latent.R @@ -435,7 +435,7 @@ test_that(paste( "get_refmodel() with `latent = TRUE` and without `dis` gives no message", "about the `NA` dispersion" ), { - tstsetups <- grep("^rstanarm\\.glm\\.poiss\\.", names(fits), value = TRUE) + tstsetups <- grep("\\.glm\\.poiss\\.", names(fits), value = TRUE) skip_if(length(tstsetups) == 0) for (tstsetup in tstsetups) { expect_no_message( From ef0bcf6b9f51487f02c0eeef0b91f54b801b22fe Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:44:49 +0200 Subject: [PATCH 26/40] Use a Poisson fit without observation weights in the NA dispersion test --- tests/testthat/test_latent.R | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/testthat/test_latent.R b/tests/testthat/test_latent.R index d5c41b2fd..0db4be179 100644 --- a/tests/testthat/test_latent.R +++ b/tests/testthat/test_latent.R @@ -435,13 +435,16 @@ test_that(paste( "get_refmodel() with `latent = TRUE` and without `dis` gives no message", "about the `NA` dispersion" ), { - tstsetups <- grep("\\.glm\\.poiss\\.", names(fits), value = TRUE) - skip_if(length(tstsetups) == 0) - for (tstsetup in tstsetups) { - expect_no_message( - refm <- get_refmodel(fits[[tstsetup]], latent = TRUE), - message = "will consist of only `NA`s" - ) - expect_true(all(is.na(refm$dis)), info = tstsetup) - } + skip_if_not_installed("rstanarm") + set.seed(4521) + dat <- data.frame(y = rpois(30, 2), x1 = rnorm(30), x2 = rnorm(30)) + fit <- suppressWarnings(rstanarm::stan_glm( + y ~ x1 + x2, family = poisson(), data = dat, chains = 1, iter = 200, + refresh = 0, seed = 7412 + )) + expect_no_message( + refm <- get_refmodel(fit, latent = TRUE), + message = "will consist of only `NA`s" + ) + expect_true(all(is.na(refm$dis))) }) From 46e9d5ff831a6a47382d9aa2e73259d61044b892 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 10:52:10 +0200 Subject: [PATCH 27/40] Leave out the sigma column of as.matrix() for a latent projection without dispersion --- R/methods.R | 10 +++++++++- man/as.matrix.projection.Rd | 4 ++++ man/as_draws_matrix.projection.Rd | 4 ++++ tests/testthat/test_latent.R | 10 ++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/R/methods.R b/R/methods.R index a7cbfc5c5..d89d76014 100644 --- a/R/methods.R +++ b/R/methods.R @@ -2528,6 +2528,10 @@ get_subparams.mmblogit <- function(x, ...) { #' reference model parameters. For a submodel with smooth and group-level #' terms (GAMM), the projected draws cannot be extracted yet (error). #' +#' For the latent projection, column `sigma` contains the projected latent +#' dispersion. If `$dis` consists of only `NA`s (see argument `dis` +#' of [init_refmodel()]), there is no column `sigma`. +#' #' @return An \eqn{S_{\mathrm{prj}} \times Q}{S_prj x Q} matrix of projected #' draws, with \eqn{S_{\mathrm{prj}}}{S_prj} denoting the number of projected #' draws and \eqn{Q} the number of parameters. If `allow_nonconst_wdraws_prj` @@ -2598,7 +2602,11 @@ as.matrix.projection <- function(x, nm_scheme = NULL, } stopifnot(nm_scheme %in% c("rstanarm", "brms")) res <- do.call(rbind, lapply(x$outdmin, get_subparams, nm_scheme = nm_scheme)) - if (x$refmodel$family$family == "gaussian") res <- cbind(res, sigma = x$dis) + # Without a latent dispersion (all `NA`s), there is no `sigma` column: + if (x$refmodel$family$family == "gaussian" && + !(isTRUE(x$refmodel$family$for_latent) && all(is.na(x$dis)))) { + res <- cbind(res, sigma = x$dis) + } if (!x[["const_wdraws_prj"]]) { attr(res, "wdraws_prj") <- x[["wdraws_prj"]] } diff --git a/man/as.matrix.projection.Rd b/man/as.matrix.projection.Rd index 8fa2446a1..c31a207a7 100644 --- a/man/as.matrix.projection.Rd +++ b/man/as.matrix.projection.Rd @@ -61,6 +61,10 @@ reference model fit can use another basis (\code{\link[rstanarm:stan_gamm4]{rsta \pkg{brms} do so for \code{s()} terms). Then these columns do not match the reference model parameters. For a submodel with smooth and group-level terms (GAMM), the projected draws cannot be extracted yet (error). + +For the latent projection, column \code{sigma} contains the projected latent +dispersion. If \verb{$dis} consists of only \code{NA}s (see argument \code{dis} +of \code{\link[=init_refmodel]{init_refmodel()}}), there is no column \code{sigma}. } \examples{ \dontshow{if (requireNamespace("rstanarm", quietly = TRUE)) withAutoprint(\{ # examplesIf} diff --git a/man/as_draws_matrix.projection.Rd b/man/as_draws_matrix.projection.Rd index abcb5d5ee..d9449ae4c 100644 --- a/man/as_draws_matrix.projection.Rd +++ b/man/as_draws_matrix.projection.Rd @@ -56,6 +56,10 @@ reference model fit can use another basis (\code{\link[rstanarm:stan_gamm4]{rsta \pkg{brms} do so for \code{s()} terms). Then these columns do not match the reference model parameters. For a submodel with smooth and group-level terms (GAMM), the projected draws cannot be extracted yet (error). + +For the latent projection, column \code{sigma} contains the projected latent +dispersion. If \verb{$dis} consists of only \code{NA}s (see argument \code{dis} +of \code{\link[=init_refmodel]{init_refmodel()}}), there is no column \code{sigma}. } \examples{ \dontshow{if (requireNamespace("rstanarm", quietly = TRUE)) withAutoprint(\{ # examplesIf} diff --git a/tests/testthat/test_latent.R b/tests/testthat/test_latent.R index 0db4be179..6c70dccfa 100644 --- a/tests/testthat/test_latent.R +++ b/tests/testthat/test_latent.R @@ -447,4 +447,14 @@ test_that(paste( message = "will consist of only `NA`s" ) expect_true(all(is.na(refm$dis))) + # Without a latent dispersion, as.matrix() has no `sigma` column: + prj <- project(refm, predictor_terms = "x1", ndraws = 20, verbose = 0) + expect_false("sigma" %in% colnames(as.matrix(prj))) + # With a latent dispersion, it has one: + refm_dis <- suppressMessages( + get_refmodel(fit, latent = TRUE, dis = rep(1, length(refm$dis))) + ) + prj_dis <- project(refm_dis, predictor_terms = "x1", ndraws = 20, + verbose = 0) + expect_true("sigma" %in% colnames(as.matrix(prj_dis))) }) From b20ac2966214e0d89da269768055ca32fda14561 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 11:03:48 +0200 Subject: [PATCH 28/40] Check the reference model dispersion for the sigma column, and test as_draws_matrix() too --- NEWS.md | 2 +- R/methods.R | 5 +++-- tests/testthat/test_latent.R | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 105cb44c6..e28c211ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -44,7 +44,7 @@ If you read this from a place other than $dis` is all `NA` (latent projection), there is no `sigma` + # column: if (x$refmodel$family$family == "gaussian" && - !(isTRUE(x$refmodel$family$for_latent) && all(is.na(x$dis)))) { + !(x$refmodel$family$for_latent && all(is.na(x$refmodel$dis)))) { res <- cbind(res, sigma = x$dis) } if (!x[["const_wdraws_prj"]]) { diff --git a/tests/testthat/test_latent.R b/tests/testthat/test_latent.R index 6c70dccfa..73a533aaf 100644 --- a/tests/testthat/test_latent.R +++ b/tests/testthat/test_latent.R @@ -433,7 +433,7 @@ test_that(paste( test_that(paste( "get_refmodel() with `latent = TRUE` and without `dis` gives no message", - "about the `NA` dispersion" + "about the `NA` dispersion, and as.matrix.projection() has no `sigma` column" ), { skip_if_not_installed("rstanarm") set.seed(4521) @@ -450,6 +450,9 @@ test_that(paste( # Without a latent dispersion, as.matrix() has no `sigma` column: prj <- project(refm, predictor_terms = "x1", ndraws = 20, verbose = 0) expect_false("sigma" %in% colnames(as.matrix(prj))) + expect_false( + "sigma" %in% posterior::variables(posterior::as_draws_matrix(prj)) + ) # With a latent dispersion, it has one: refm_dis <- suppressMessages( get_refmodel(fit, latent = TRUE, dis = rep(1, length(refm$dis))) From 4ef6c63d0a37a3b8c2fac75be4e73140e8c59c1c Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 15:30:46 +0200 Subject: [PATCH 29/40] Use one simulated dataset and cv_varsel() in the Poisson and negative binomial examples, and decide the survival submodel sizes from the plots --- vignettes/latent.Rmd | 292 ++++++++++++------------------------------- 1 file changed, 83 insertions(+), 209 deletions(-) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index c360e160f..7ab5fd49c 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -63,24 +63,22 @@ The families currently supported by **projpred**'s augmented-data projection are ## Example: Poisson distribution In this example, we will illustrate that in case of a family supported by **projpred**'s traditional projection (here the `poisson()` family), the latent projection can reduce the runtime of the variable selection compared to the traditional projection. -We first use L1 search (see argument `method` of `varsel()` and `cv_varsel()`) only to show an issue of the traditional projection. -In general, forward search (the default) is more accurate than L1 search. ### Data -First, we generate a training and a test dataset with a Poisson-distributed response: +First, we generate a dataset with a Poisson-distributed response. +The linear predictor depends on 4 continuous predictors and 1 categorical predictor, and there are 20 noise predictors: ```{r dat_poiss} -# Number of observations in the training dataset (= number of observations in -# the test dataset): -N <- 71 +# Number of observations: +N <- 100 # Data-generating function: -sim_poiss <- function(nobs = 2 * N, ncon = 10, ncats = 4, nnoise = 39) { +sim_poiss <- function(nobs = N, ncon = 4, ncats = 4, nnoise = 20) { # Regression coefficients for continuous predictors: - coefs_con <- rnorm(ncon) + coefs_con <- seq(from = 0.2, to = 0.8, length.out = ncon) # Continuous predictors: dat_sim <- matrix(rnorm(nobs * ncon), ncol = ncon) # Start linear predictor: - linpred <- 2.1 + dat_sim %*% coefs_con + linpred <- 1.4 + dat_sim %*% coefs_con # Categorical predictor: dat_sim <- data.frame( @@ -89,7 +87,7 @@ sim_poiss <- function(nobs = 2 * N, ncon = 10, ncats = 4, nnoise = 39) { labels = paste0("cat", seq_len(ncats))) ) # Regression coefficients for the categorical predictor: - coefs_cat <- rnorm(ncats) + coefs_cat <- seq(from = 0.2, to = 0.8, length.out = ncats) # Continue linear predictor: linpred <- linpred + coefs_cat[dat_sim$xcat] @@ -110,8 +108,6 @@ sim_poiss <- function(nobs = 2 * N, ncon = 10, ncats = 4, nnoise = 39) { # Generate data: set.seed(300417) dat_poiss <- sim_poiss() -dat_poiss_train <- head(dat_poiss, N) -dat_poiss_test <- tail(dat_poiss, N) ``` ### Reference model @@ -122,13 +118,13 @@ library(rstanarm) ``` ```{r ref_fit_poiss} # Number of regression coefficients: -( D <- sum(grepl("^x", names(dat_poiss_train))) ) +( D <- sum(grepl("^x", names(dat_poiss))) ) # Prior guess for the number of relevant (i.e., non-zero) regression # coefficients: -p0 <- 10 +p0 <- 5 # Prior guess for the overall magnitude of the response values, see Table 1 of # Piironen and Vehtari (2017, DOI: 10.1214/17-EJS1337SI): -mu_prior <- 100 +mu_prior <- 10 # Hyperprior scale for tau, the global shrinkage parameter: tau0 <- p0 / (D - p0) / sqrt(mu_prior) / sqrt(N) # Set this manually if desired: @@ -143,7 +139,7 @@ refm_fml <- y ~ . refm_fit_poiss <- stan_glm( formula = refm_fml, family = poisson(), - data = dat_poiss_train, + data = dat_poiss, prior = hs(global_scale = tau0, slab_df = 100, slab_scale = 1), ### Only for the sake of speed (not recommended in general): chains = 2, iter = 1000, @@ -153,74 +149,55 @@ refm_fit_poiss <- stan_glm( refresh = 0 ) ``` -Due to the technical reasons for which we reduced `chains` and `iter` in this vignette, we ignore the bulk-ESS warning here. ### Variable selection using the latent projection Within **projpred**, we define the reference model object explicitly and set `latent = TRUE` in the corresponding `get_refmodel()` call (see section ["Implementation"](#impl)) so that the latent projection is used in downstream functions. -Since we have a hold-out test dataset available, we can use `varsel()` with argument `d_test` instead of `cv_varsel()`. +We run `cv_varsel()` with its defaults: PSIS-LOO CV and validated search. Furthermore, we measure the runtime to be able to compare it to the traditional projection's later: ```{r projpred_attach, message=FALSE} library(projpred) ``` -```{r vs_lat} -d_test_lat_poiss <- list( - data = dat_poiss_test, - offset = rep(0, nrow(dat_poiss_test)), - weights = rep(1, nrow(dat_poiss_test)), - ### Here, we are not interested in latent-scale post-processing, so we can set - ### element `y` to a vector of `NA`s: - y = rep(NA, nrow(dat_poiss_test)), - ### - y_oscale = dat_poiss_test$y -) +```{r cvvs_lat} refm_poiss <- get_refmodel(refm_fit_poiss, latent = TRUE) -time_lat <- system.time(vs_lat <- varsel( +time_lat <- system.time(cvvs_lat <- cv_varsel( refm_poiss, - d_test = d_test_lat_poiss, - ### Only for demonstrating an issue with the traditional projection in the - ### next step (not recommended in general): - method = "L1", - ### - ### Only for the sake of speed (not recommended in general): - nclusters_pred = 20, - ### - nterms_max = 14, ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0, ### - ### For comparability with varsel() based on the traditional projection: seed = 95930 - ### )) ``` ```{r time_lat} print(time_lat) ``` +The warnings about high Pareto $\hat{k}$ values concern only a few of the `r if (!params$EVAL) character() else N` observations. +With the short MCMC run of this vignette (1000 posterior draws), the threshold for a reliable Pareto $\hat{k}$ is about 0.67 [@vehtari_pareto_2022]. +In a real analysis, we would run more MCMC iterations, check the Pareto $\hat{k}$ values again, and use $K$-fold CV (`cv_method = "kfold"`) if many of them are high. In order to decide for a submodel size, we first inspect the `plot()` results. In contrast to the main vignette where we used the mean log predictive density (MLPD) as predictive performance statistic for a `gaussian()` family reference model (and `gaussian()` submodels), we have a discrete family (`poisson()`) here, so it makes sense to exponentiate the MLPD to obtain the geometric mean predictive density (GMPD; in case of a discrete response, the predictive density values are actually predictive *probabilities* and hence the GMPD is bounded by 0 and 1). As in the main vignette, we plot with `deltas = TRUE` (in case of the GMPD, this means that the *ratio* of the submodel GMPD vs. the reference model GMPD is shown). Via global option `projpred.plot_vsel_size_position`, we set argument `size_position` of `plot.vsel()` to `"secondary_x"` to make the submodel sizes readable in all of the plots in this vignette. -```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the latent projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio is close to 0 at small sizes and then rises to the reference model."} +```{r plot_vsel_lat, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the latent projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio increases with the size, reaches the reference model and then levels off."} ggplot2::theme_set(bayesplot::theme_default(base_family = "sans")) options(projpred.plot_vsel_size_position = "secondary_x") -( gg_lat <- plot(vs_lat, stats = "gmpd", deltas = TRUE) ) +( gg_lat <- plot(cvvs_lat, stats = "gmpd", deltas = TRUE) ) ``` -Based on this plot, we decide for a submodel size of 11: +Based on this plot, we decide for a submodel size of 5: ```{r size_man_lat} -size_decided_lat <- 11 +size_decided_lat <- 5 ``` This is also the size that `suggest_size()` would suggest: ```{r size_sgg_lat} -suggest_size(vs_lat, stat = "gmpd") +suggest_size(cvvs_lat, stat = "gmpd") ``` In the predictor ranking up to the selected size of `r if (!params$EVAL) character() else size_decided_lat`, we can see that **projpred** has correctly selected the truly relevant predictors first and only then the noise predictors: ```{r predictors_final_lat} -rk_lat <- ranking(vs_lat) +rk_lat <- ranking(cvvs_lat) ( predictors_final_lat <- head(rk_lat[["fulldata"]], size_decided_lat) ) ``` @@ -228,128 +205,34 @@ We will skip post-selection inference here (see the main vignette for a demonstr ### Variable selection using the traditional projection -We will now look at what **projpred**'s traditional projection would have given. -For this, we increase `nterms_max` because this will reveal an issue with this approach: -```{r suppress_warn_poiss, include=FALSE} -warn_instable_orig <- options(projpred.warn_instable_projections = FALSE) -``` -```{r vs_trad} -d_test_trad_poiss <- d_test_lat_poiss -d_test_trad_poiss$y <- d_test_trad_poiss$y_oscale -d_test_trad_poiss$y_oscale <- NULL -time_trad <- system.time(vs_trad <- varsel( - refm_fit_poiss, - d_test = d_test_trad_poiss, - ### Only for demonstrating an issue with the traditional projection (not - ### recommended in general): - method = "L1", - ### - ### Only for the sake of speed (not recommended in general): - nclusters_pred = 20, - ### - nterms_max = 30, - ### In interactive use, we recommend not to deactivate the verbose mode: - verbose = 0, - ### - ### For comparability with varsel() based on the latent projection: - seed = 95930 - ### -)) -``` -```{r unsuppress_warn_poiss, include=FALSE} -options(warn_instable_orig) -rm(warn_instable_orig) -``` -```{r post_vs_trad, fig.alt="The same plot for the traditional projection. Several noise terms enter the predictor ranking before relevant ones, and the submodels reach the reference model only at a much larger size."} -print(time_trad) -( gg_trad <- plot(vs_trad, stats = "gmpd", deltas = TRUE) ) -``` -As these results show, the traditional projection takes longer than the latent projection, although the difference is rather small on absolute scale (which is due to the fact that the L1 search is already quite fast). -More importantly however, the predictor ranking contains several noise terms before truly relevant ones, causing the predictive performance of the reference model not to be reached before submodel size 28. - -### Forward search - -In general, forward search is more accurate than L1 search (see section "Details" in `?varsel`). -We now run `varsel()` with forward search (the default `method`) for both projections: -```{r vs_fw_poiss} -time_lat_fw <- system.time(vs_lat_fw <- varsel( - refm_poiss, - d_test = d_test_lat_poiss, - ### Only for the sake of speed (not recommended in general): - nclusters_pred = 20, - ### - nterms_max = 14, - ### In interactive use, we recommend not to deactivate the verbose mode: - verbose = 0, - ### - seed = 95930 -)) -time_trad_fw <- system.time(vs_trad_fw <- varsel( +We now run the same `cv_varsel()` call with **projpred**'s traditional projection. +For a Poisson submodel, each projection is an iterative fit, and validated search repeats the search for each of the `r if (!params$EVAL) character() else N` LOO CV folds. +To keep the runtime of this vignette short, we use fewer clusters in the performance evaluation (`nclusters_pred = 20`) and search only up to size 10 (`nterms_max = 10`): +```{r cvvs_trad} +time_trad <- system.time(cvvs_trad <- cv_varsel( refm_fit_poiss, - d_test = d_test_trad_poiss, ### Only for the sake of speed (not recommended in general): nclusters_pred = 20, + nterms_max = 10, ### - nterms_max = 14, - ### In interactive use, we recommend not to deactivate the verbose mode: - verbose = 0, - ### - seed = 95930 -)) -rbind(latent = time_lat_fw, traditional = time_trad_fw)[, "elapsed"] -ranking(vs_lat_fw)[["fulldata"]] -ranking(vs_trad_fw)[["fulldata"]] -``` -With forward search, both projections rank the 11 truly relevant predictor terms first (in a slightly different order). -The latent projection is much faster (see the elapsed times in seconds above). - -### LOO CV with validated search - -Without a hold-out test dataset, we use `cv_varsel()` with PSIS-LOO CV and validated search (the defaults of `cv_varsel()`). -For the latent projection, this runs fast: -```{r cvvs_lat_poiss} -time_cvvs_lat <- system.time(cvvs_lat_poiss <- cv_varsel( - refm_poiss, - nterms_max = 14, ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0, ### seed = 95930 )) -print(time_cvvs_lat) -``` -```{r plot_cvvs_lat_poiss, fig.alt="The same kind of plot for LOO CV with validated search and the latent projection of the Poisson model. The ratio is close to 0 at small sizes and then rises to the reference model."} -plot(cvvs_lat_poiss, stats = "gmpd", deltas = TRUE) ``` -```{r rk_cvvs_lat_poiss} -suggest_size(cvvs_lat_poiss, stat = "gmpd") -head(ranking(cvvs_lat_poiss)[["fulldata"]], 11) -``` -`suggest_size()` gives size 11, and the first 11 predictor terms of the full-data predictor ranking are the truly relevant ones. -The warnings about high Pareto $\hat{k}$ values (19 of the 71 observations) mean that PSIS-LOO CV may not be reliable here, probably because of the small number of observations and the short MCMC run of this vignette. -In a real analysis, we would then use $K$-fold CV (`cv_method = "kfold"`). - -For the traditional projection, each projection onto a Poisson submodel is an iterative fit, and validated search repeats the search for each of the 71 LOO CV folds, so the same call takes much longer. -In a test run, the call took about 100 times as long as the latent projection (about 390 seconds compared to about 4 seconds). -It ranked the same 11 truly relevant predictor terms first, and `suggest_size()` gave size 12. -Therefore, we do not run it here: -```{r cvvs_trad_poiss, eval=FALSE} -cvvs_trad_poiss <- cv_varsel( - refm_fit_poiss, - nterms_max = 14, - ### In interactive use, we recommend not to deactivate the verbose mode: - verbose = 0, - ### - seed = 95930 -) +```{r post_vs_trad} +rbind(latent = time_lat, traditional = time_trad)[, "elapsed"] +suggest_size(cvvs_trad, stat = "gmpd") +head(ranking(cvvs_trad)[["fulldata"]], size_decided_lat) ``` +The traditional projection takes much longer than the latent projection (see the elapsed times in seconds above), although it uses the smaller settings. +It selects the same `r if (!params$EVAL) character() else size_decided_lat` predictor terms first. ### Conclusion This example showed that the latent projection can be advantageous also for families supported by **projpred**'s traditional projection. -With L1 search, which we used only to show an issue of the traditional projection, the traditional projection ranked several noise terms before truly relevant ones. -With forward search, both projections ranked the 11 truly relevant predictor terms first. -The latent projection was much faster, in particular in LOO CV with validated search. +Both projections selected the truly relevant predictor terms first, but the latent projection was much faster. ## Example: Negative binomial distribution {#negbinex} @@ -368,7 +251,7 @@ However, since Table 1 of @piironen_sparsity_2017 does not list the negative bin refm_fit_nebin <- stan_glm( formula = refm_fml, family = neg_binomial_2(), - data = dat_poiss_train, + data = dat_poiss, prior = hs(global_scale = tau0, slab_df = 100, slab_scale = 1), ### Only for the sake of speed (not recommended in general): chains = 2, iter = 1000, @@ -376,12 +259,12 @@ refm_fit_nebin <- stan_glm( refresh = 0 ) ``` -Again, we ignore the bulk-ESS warning due to the technical reasons for which we reduced `chains` and `iter` in this vignette. +If a warning about a low effective sample size (ESS) appears, we ignore it here because of the technical reasons for which we reduced `chains` and `iter` in this vignette. ### Variable selection using the latent projection To request the latent projection with `latent = TRUE`, we now need to specify more arguments (`latent_ll_oscale` and `latent_ppd_oscale`; the internal default for `latent_ilink` works correctly in this example) which will be passed to `extend_family()`^[The suffix `_prec` in `refm_prec` stands for "precision" because here, we follow the Stan convention (see the Stan documentation for the `neg_binomial_2` distribution, the `brms::negbinomial()` documentation, and the [**brms**](https://paulbuerkner.com/brms/) vignette ["Parameterization of Response Distributions in brms"](https://paulbuerkner.com/brms/articles/brms_families.html)) and prefer the term *precision* parameter for what is denoted by $\phi$ there (confusingly, argument `size` in `?stats::NegBinomial`---which is the same as $\phi$ from the Stan notation---is called the *dispersion* parameter there, although the variance is increased by its reciprocal).]: -```{r vs_nebin} +```{r cvvs_nebin} refm_prec <- as.matrix(refm_fit_nebin)[, "reciprocal_dispersion", drop = FALSE] latent_ll_oscale_nebin <- function(ilpreds, dis = rep(NA, nrow(ilpreds)), @@ -414,13 +297,8 @@ latent_ppd_oscale_nebin <- function(ilpreds_resamp, refm_nebin <- get_refmodel(refm_fit_nebin, latent = TRUE, latent_ll_oscale = latent_ll_oscale_nebin, latent_ppd_oscale = latent_ppd_oscale_nebin) -vs_nebin <- varsel( +cvvs_nebin <- cv_varsel( refm_nebin, - d_test = d_test_lat_poiss, - ### Only for the sake of speed (not recommended in general): - nclusters_pred = 20, - ### - nterms_max = 14, ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0 ### @@ -428,24 +306,24 @@ vs_nebin <- varsel( ``` Again, we first inspect the `plot()` results to decide for a submodel size: -```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, rises above 1 and then levels off. The first 11 predictors of the ranking are the 11 truly relevant ones."} -( gg_nebin <- plot(vs_nebin, stats = "gmpd", deltas = TRUE) ) +```{r plot_vsel_nebin, fig.alt="The same kind of plot for the latent projection of the negative binomial model. The ratio increases with the size, rises slightly above 1 and then slowly decreases towards 1. The first 5 predictors of the ranking are the 5 truly relevant ones."} +( gg_nebin <- plot(cvvs_nebin, stats = "gmpd", deltas = TRUE) ) ``` -For the decision of the final submodel size, we act as if we preferred accuracy over sparsity in their trade-off mentioned in the [main vignette](https://mc-stan.org/projpred/articles/projpred.html#decision-size), so we decide for a submodel size of 11: +Based on this plot, we decide for a submodel size of 5: ```{r size_man_nebin} -size_decided_nebin <- 11 +size_decided_nebin <- 5 ``` -This is not the size that `suggest_size()` would suggest, but as mentioned in the main vignette and in the documentation, `suggest_size()` provides only a quite heuristic decision (so we stick with our manual decision here): +This is also the size that `suggest_size()` would suggest: ```{r size_sgg_nebin} -suggest_size(vs_nebin, stat = "gmpd") +suggest_size(cvvs_nebin, stat = "gmpd") ``` As we can see from the predictor ranking included in the plot, our selected `r if (!params$EVAL) character() else size_decided_nebin` predictor terms are exactly the truly relevant ones. More explicitly, our selected predictor terms are: ```{r predictors_final_nebin} -rk_nebin <- ranking(vs_nebin) +rk_nebin <- ranking(cvvs_nebin) ( predictors_final_nebin <- head(rk_nebin[["fulldata"]], size_decided_nebin) ) ``` @@ -502,7 +380,7 @@ refm_fit_weib <- brms::brm( formula = yobs | cens(is_censored) ~ ., family = brms::weibull(), data = dat_sim_weib, - prior = brms::prior(R2D2(mean_R2 = 0.4, prec_R2 = 2.5, cons_D2 = 1)), + prior = brms::prior(R2D2(mean_R2 = 1/3, prec_R2 = 3)), ### Only for the sake of speed (not recommended in general): chains = 2, ### @@ -583,28 +461,14 @@ For K-fold CV, see section ["K-fold CV with custom response-scale functions"](#k Run `cv_varsel()`: ```{r weibull_cvvs} -# For running projpred's CV in parallel (see cv_varsel()'s argument `parallel`): -# Note: Parallel processing is disabled during package building to avoid issues -use_parallel <- FALSE # Set to TRUE for actual parallel processing -if (use_parallel) { - future::plan(future.mirai::mirai_multisession, workers = ncores) -} cvvs_weib <- cv_varsel( refm_weib, - ### Only for the sake of speed (not recommended in general): - nterms_max = 11, - ### - parallel = use_parallel, ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0 - ### ) -# Tear down the CV parallelization setup: -if (use_parallel) { - future::plan(future::sequential) -} ``` -In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). +As in the Poisson example, warnings about high Pareto $\hat{k}$ values can come from the short MCMC run of this vignette. +In a real analysis, we would run more MCMC iterations, check the Pareto $\hat{k}$ values again, and use $K$-fold CV (`cv_method = "kfold"`) if many of them are high. Plot the results: ```{r weibull_plot_cvvs, fig.alt="Plot of the MLPD difference between each submodel and the reference model, with uncertainty bars, for the Weibull model with right-censored observations. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. The first 10 predictors are the 10 truly relevant ones. The difference increases with the size and then levels off close to zero (dashed line)."} @@ -612,9 +476,20 @@ plot(cvvs_weib, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. -Project onto the submodel consisting of the first `n_pred_truth` predictors and perform a "posterior-projection predictive check" (PPPC): +Based on this plot, we decide for a submodel size of 10: +```{r weibull_size} +size_decided_weib <- 10 +``` + +This is not the size that `suggest_size()` would suggest, but as mentioned in the main vignette and in the documentation, `suggest_size()` provides only a quite heuristic decision (so we stick with our manual decision here): +```{r weibull_size_sgg} +suggest_size(cvvs_weib, stat = "mlpd") +``` + +Project onto the submodel of this size and perform a "posterior-projection predictive check" (PPPC): ```{r weibull_pppc, fig.alt="Kaplan-Meier overlay plot with the time on the x-axis and the survival probability on the y-axis: the Kaplan-Meier curve of the observed times as a dark line, and the Kaplan-Meier curves of predictive draws from the Weibull submodel as light lines. The light lines are close to the dark line."} -predictors_final_weib <- head(ranking(cvvs_weib)[["fulldata"]], n_pred_truth) +predictors_final_weib <- head(ranking(cvvs_weib)[["fulldata"]], + size_decided_weib) prj_weib <- project(refm_weib, predictor_terms = predictors_final_weib) prj_predict_weib <- proj_predict(prj_weib) bayesplot::ppc_km_overlay(y = dat_sim_weib$yobs, yrep = prj_predict_weib, @@ -643,7 +518,7 @@ refm_fit_lnorm <- brms::brm( formula = yobs | cens(is_censored) ~ ., family = brms::lognormal(), data = dat_sim_lnorm, - prior = brms::prior(R2D2(mean_R2 = 0.4, prec_R2 = 2.5, cons_D2 = 1)), + prior = brms::prior(R2D2(mean_R2 = 1/3, prec_R2 = 3)), ### Only for the sake of speed (not recommended in general): chains = 2, ### @@ -709,39 +584,27 @@ latent_ppd_oscale_lnorm <- function( refm_lnorm <- get_refmodel( refm_fit_lnorm, latent = TRUE, + latent_ilink = function(lpreds, cl_ref, + wdraws_ref = rep(1, length(cl_ref))) lpreds, latent_ll_oscale = latent_ll_oscale_lnorm, latent_ppd_oscale = latent_ppd_oscale_lnorm, dis = as.matrix(refm_fit_lnorm)[, "sigma", drop = FALSE] ) ``` -The message concerning `latent_ilink` can be safely ignored here (the internal default based on `family$linkinv` works correctly in this case). +Argument `latent_ilink` returns the latent predictors unchanged, because the two custom functions use them as `meanlog`. Argument `dis` contains the `sigma` draws of the full-data fit. For K-fold CV, see section ["K-fold CV with custom response-scale functions"](#kfold-custom). Run `cv_varsel()`: ```{r lognormal_cvvs} -# For running projpred's CV in parallel (see cv_varsel()'s argument `parallel`): -# Note: Parallel processing is disabled during package building to avoid issues -use_parallel <- FALSE # Set to TRUE for actual parallel processing -if (use_parallel) { - future::plan(future.mirai::mirai_multisession, workers = ncores) -} cvvs_lnorm <- cv_varsel( refm_lnorm, - ### Only for the sake of speed (not recommended in general): - nterms_max = 11, - ### - parallel = use_parallel, ### In interactive use, we recommend not to deactivate the verbose mode: verbose = 0 - ### ) -# Tear down the CV parallelization setup: -if (use_parallel) { - future::plan(future::sequential) -} ``` -In this case, we will ignore the warnings about high Pareto-$\hat{k}$ values because we chose quite rough settings (e.g., only 2 MCMC chains) in this vignette (for technical reasons). +As in the Poisson example, warnings about high Pareto $\hat{k}$ values can come from the short MCMC run of this vignette. +In a real analysis, we would run more MCMC iterations, check the Pareto $\hat{k}$ values again, and use $K$-fold CV (`cv_method = "kfold"`) if many of them are high. Plot the results: ```{r lognormal_plot_cvvs, fig.alt="The same kind of plot for the log-normal model with right-censored observations. The first 10 predictors are the 10 truly relevant ones. The difference increases with the size and then levels off close to zero (dashed line)."} @@ -749,9 +612,20 @@ plot(cvvs_lnorm, stats = "mlpd", deltas = TRUE) ``` Hence, the truly relevant predictors are identified correctly. -Project onto the submodel consisting of the first `n_pred_truth` predictors and perform a "posterior-projection predictive check" (PPPC): +Based on this plot, we decide for a submodel size of 10: +```{r lognormal_size} +size_decided_lnorm <- 10 +``` + +This is also the size that `suggest_size()` would suggest: +```{r lognormal_size_sgg} +suggest_size(cvvs_lnorm, stat = "mlpd") +``` + +Project onto the submodel of this size and perform a "posterior-projection predictive check" (PPPC): ```{r lognormal_pppc, fig.alt="The same Kaplan-Meier overlay plot for the log-normal submodel. The light lines are close to the dark line."} -predictors_final_lnorm <- head(ranking(cvvs_lnorm)[["fulldata"]], n_pred_truth) +predictors_final_lnorm <- head(ranking(cvvs_lnorm)[["fulldata"]], + size_decided_lnorm) prj_lnorm <- project(refm_lnorm, predictor_terms = predictors_final_lnorm) prj_predict_lnorm <- proj_predict(prj_lnorm) bayesplot::ppc_km_overlay(y = dat_sim_lnorm$yobs, yrep = prj_predict_lnorm, From be984bd71e104a50c7d94da5539d8fc1581b0655 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 15:30:52 +0200 Subject: [PATCH 30/40] Remove the pointer to the L1 issue of the old Poisson example --- vignettes/projpred.Rmd | 1 - 1 file changed, 1 deletion(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 45ab72e38..ce3238144 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -534,7 +534,6 @@ Some speed-up possibilities are: 1. Using L1 search (see argument `method` of `varsel()` or `cv_varsel()`) instead of forward search. Note that L1 search implies `nclusters = 1` and is not always supported. In general, forward search is more accurate than L1 search and hence more desirable (see section "Details" in `?varsel` or `?cv_varsel` for a more detailed comparison of the two). - The issue demonstrated in the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution) is related to this. 1. Setting argument `refit_prj` (of `varsel()` or `cv_varsel()`) to `FALSE`, which basically means to set `ndraws_pred = ndraws` and `nclusters_pred = nclusters`, but in a more efficient (i.e., faster) way. In case of L1 search, this means that the L1-penalized projections of the regression coefficients are used for the predictive performance evaluation, which is usually undesired [@piironen_projective_2020, section 4]. From ea8f4becc8249e01bf28a7d904c4f945c9b1d1df Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 15:48:22 +0200 Subject: [PATCH 31/40] Plot the results of the traditional projection in the Poisson example --- vignettes/latent.Rmd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vignettes/latent.Rmd b/vignettes/latent.Rmd index 7ab5fd49c..2fd53eaec 100644 --- a/vignettes/latent.Rmd +++ b/vignettes/latent.Rmd @@ -226,6 +226,11 @@ rbind(latent = time_lat, traditional = time_trad)[, "elapsed"] suggest_size(cvvs_trad, stat = "gmpd") head(ranking(cvvs_trad)[["fulldata"]], size_decided_lat) ``` + +```{r plot_vsel_trad, fig.alt="Plot of the ratio of the geometric mean predictive density (GMPD) of each submodel to that of the reference model, with uncertainty bars, for the traditional projection of the Poisson model. The top x-axis shows the submodel size and the bottom x-axis the predictor added at each size. A dashed horizontal line at 1 marks the reference model. The ratio increases with the size, reaches the reference model and then levels off."} +( gg_trad <- plot(cvvs_trad, stats = "gmpd", deltas = TRUE) ) +``` + The traditional projection takes much longer than the latent projection (see the elapsed times in seconds above), although it uses the smaller settings. It selects the same `r if (!params$EVAL) character() else size_decided_lat` predictor terms first. From e149c5fa56a398dc1431a2c1678c9b73795a9d84 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 15:48:27 +0200 Subject: [PATCH 32/40] Move the plot theme setting of the main vignette --- vignettes/projpred.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index ce3238144..d75e0a20b 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -177,6 +177,7 @@ Since the difference-scale uncertainty is usually more helpful than the actual-s The search ran up to submodel size 19 (here, the default of argument `nterms_max` of `cv_varsel()`). To keep the labels readable, the plot shows only the sizes up to 12 (argument `nterms_max` of `plot.vsel()`): ```{r plot_vsel, fig.alt="Plot of the MLPD difference between each submodel and the reference model against the submodel size from 0 to 12, with uncertainty bars. Below each size, the x-axis shows the predictor that the full-data search added and the proportion of CV folds that put this predictor at the same position. A dashed horizontal line at zero marks the reference model. The difference reaches zero at size 7 and stays slightly above zero for larger sizes."} +ggplot2::theme_set(bayesplot::theme_default(base_family = "sans")) options(projpred.plot_vsel_size_position = "primary_x_bottom") options(projpred.plot_vsel_text_angle = 0) options(projpred.plot_vsel_show_cv_proportions = TRUE) @@ -332,7 +333,6 @@ A visualization of the projected posterior can be achieved with the [**bayesplot library(bayesplot) ``` ```{r bayesplot_prj, fig.alt="Interval plot of the projected posterior, with one row for the intercept, each of the 7 selected predictors and sigma, and the parameter value on the x-axis. Each row has a point for the median, a thick line for the 50 % interval and a thin line for the 90 % interval."} -ggplot2::theme_set(bayesplot::theme_default(base_family = "sans")) mcmc_intervals(prj_mat) + ggplot2::coord_cartesian(xlim = c(-1.5, 1.6)) ``` From f43fa6fc53382c9ff8336ce083abc655fa9df355 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 17:08:54 +0200 Subject: [PATCH 33/40] Update the troubleshooting text on the multilevel projections for lme4 2.1-1 and the Laplace fitters --- vignettes/projpred.Rmd | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index d75e0a20b..fa6c8b24d 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -481,21 +481,23 @@ Similarly, `cv_varsel()` with `validate_search = FALSE` is more prone to overfit ### Issues with the traditional projection -For multilevel binomial models, the traditional projection may not work properly and give suboptimal results, see [#353](https://github.com/stan-dev/projpred/pull/353) on GitHub (the underlying issue is described in **lme4** issue [#682](https://github.com/lme4/lme4/issues/682)). -By suboptimal results, we mean that the relevance of the group-level terms can be underestimated. -According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in that case. +For multilevel binomial and Poisson models, the projection fits the submodels to the predictions of the reference model, so the response values of these fits are not integers. +**projpred** requires **lme4** >= 2.1-1, which fits such response values with the continuous extension of the binomial and the Poisson likelihood (see **lme4** issue [#682](https://github.com/lme4/lme4/issues/682) and pull request [#1012](https://github.com/lme4/lme4/pull/1012)). -For multilevel Poisson models, the traditional projection may take very long, see [#353](https://github.com/stan-dev/projpred/pull/353). +If all group-level terms of a binomial (logit link) or Poisson (log link) submodel are group-level intercepts (`(1 | )`), **projpred** fits the submodel with its own Laplace approximation, which is faster than `lme4::glmer()`. +Submodels with other group-level terms (for example, group-level slopes) are fitted with `lme4::glmer()`, and then the projection can take long, in particular for Poisson models. According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in that case. Finally, the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution) shows that the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that the traditional projection also supports. ### Issues with the augmented-data projection -For multilevel models, the augmented-data projection seems to suffer from the same issue as the traditional projection for the binomial family (see above), i.e., it may not work properly and give suboptimal results, see [#353](https://github.com/stan-dev/projpred/pull/353) (the underlying issue is probably similar to the one described in **lme4** issue [#682](https://github.com/lme4/lme4/issues/682)). +For multilevel models, the augmented-data projection may not work properly and give suboptimal results, see [#353](https://github.com/stan-dev/projpred/pull/353). By suboptimal results, we mean that the relevance of the group-level terms can be underestimated. According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in such cases. +If the only group-level term of a `brms::cumulative()` submodel is a group-level intercept (`(1 | )`), **projpred** fits the submodel with its own Laplace approximation, which is faster than `ordinal::clmm()`. + ### Speed {#speed} For Gaussian models with the identity link and without multilevel or additive terms, and for the latent projection without such terms, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix. From 23844a087ff25232fd0ab1accc5511d83b72a881 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 17:18:54 +0200 Subject: [PATCH 34/40] Warn about multilevel binomial and Poisson models only with lme4 < 2.1-1 --- R/projpred-package.R | 3 --- R/refmodel.R | 16 ++++++---------- man/internal-global-options.Rd | 3 --- tests/testthat/test_refmodel.R | 23 +++++++++++++++++++++++ 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/R/projpred-package.R b/R/projpred-package.R index 6f769e64a..c4714d44d 100644 --- a/R/projpred-package.R +++ b/R/projpred-package.R @@ -304,9 +304,6 @@ #' meaning of these global options, see their occurrences in the codebase. With #' the exception of `projpred.warn_allrandom_dis`, these global options are #' currently used in the unit tests to deactivate these messages and warnings. -#' Global option `projpred.warn_instable_projections` is also used (invisibly) -#' in the latent vignette to suppress the corresponding warnings while -#' illustrating the underlying issue (instable projections). #' * `projpred.additional_checks`: A single logical value indicating whether to #' run some additional checks that are not necessary to be run when users call #' the corresponding \pkg{projpred} functions. Currently, these checks are diff --git a/R/refmodel.R b/R/refmodel.R index 5fceb1f52..6f50dbdc6 100644 --- a/R/refmodel.R +++ b/R/refmodel.R @@ -1270,17 +1270,13 @@ init_refmodel <- function(object, data, formula, family, ref_predfun = NULL, "properly. The latent projection may be a remedy. See section ", "\"Troubleshooting\" of the main vignette for more information." ) - } else if (family$family == "binomial") { + } else if (family$family %in% c("binomial", "poisson") && + utils::packageVersion("lme4") < "2.1.1") { warning( - "For multilevel binomial models, the traditional projection may not ", - "work properly. The latent projection may be a remedy. See section ", - "\"Troubleshooting\" of the main vignette for more information." - ) - } else if (family$family == "poisson") { - warning( - "For multilevel Poisson models, the traditional projection may take ", - "very long. The latent projection may be a remedy. See section ", - "\"Troubleshooting\" of the main vignette for more information." + "With lme4 < 2.1-1, the group-level standard deviations of ", + "multilevel binomial and Poisson submodels can be wrong. We ", + "recommend lme4 >= 2.1-1. See section \"Troubleshooting\" of the ", + "main vignette for more information." ) } } diff --git a/man/internal-global-options.Rd b/man/internal-global-options.Rd index 127037953..712d4ba0d 100644 --- a/man/internal-global-options.Rd +++ b/man/internal-global-options.Rd @@ -15,9 +15,6 @@ logical value indicating whether to throw certain messages or warnings meaning of these global options, see their occurrences in the codebase. With the exception of \code{projpred.warn_allrandom_dis}, these global options are currently used in the unit tests to deactivate these messages and warnings. -Global option \code{projpred.warn_instable_projections} is also used (invisibly) -in the latent vignette to suppress the corresponding warnings while -illustrating the underlying issue (instable projections). \item \code{projpred.additional_checks}: A single logical value indicating whether to run some additional checks that are not necessary to be run when users call the corresponding \pkg{projpred} functions. Currently, these checks are diff --git a/tests/testthat/test_refmodel.R b/tests/testthat/test_refmodel.R index 9e0180f18..7884b247f 100644 --- a/tests/testthat/test_refmodel.R +++ b/tests/testthat/test_refmodel.R @@ -641,3 +641,26 @@ test_that(paste( proj_linpred(prj_nt, newdata = dat_nt[1:5, c("x1", "x2", "y")]) ) }) + +test_that(paste( + "init_refmodel() warns about lme4 < 2.1-1 for multilevel binomial and", + "Poisson models exactly if lme4 < 2.1-1" +), { + tstsetups <- grep("\\.glmm\\.(brnll|binom|poiss)\\.", names(fits), + value = TRUE) + skip_if(length(tstsetups) == 0) + withr::local_options(list(projpred.warn_instable_projections = NULL)) + lme4_old <- utils::packageVersion("lme4") < "2.1.1" + for (tstsetup in tstsetups) { + warns <- character() + suppressMessages(withCallingHandlers( + get_refmodel(fits[[tstsetup]]), + warning = function(w) { + warns <<- c(warns, conditionMessage(w)) + invokeRestart("muffleWarning") + } + )) + expect_identical(any(grepl("With lme4 < 2.1-1", warns, fixed = TRUE)), + lme4_old, info = tstsetup) + } +}) From f16fd7c2144a643c0f7b19d3e67b1d6b9caf0fa7 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 19:01:09 +0200 Subject: [PATCH 35/40] Shorten the Troubleshooting section of the main vignette --- vignettes/projpred.Rmd | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index fa6c8b24d..672530314 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -475,20 +475,7 @@ In this case, the latent-space approach by @catalina_latent_2021 might help, see ### Overfitting {#overfitting} If `varsel()` is used, the lack of a CV in `varsel()` may lead to overconfident and overfitted results. -In this case, we recommend `cv_varsel()` instead of `varsel()` (`cv_varsel()` should be used for final results anyway). - -Similarly, `cv_varsel()` with `validate_search = FALSE` is more prone to overfitting than `cv_varsel()` with `validate_search = TRUE`. - -### Issues with the traditional projection - -For multilevel binomial and Poisson models, the projection fits the submodels to the predictions of the reference model, so the response values of these fits are not integers. -**projpred** requires **lme4** >= 2.1-1, which fits such response values with the continuous extension of the binomial and the Poisson likelihood (see **lme4** issue [#682](https://github.com/lme4/lme4/issues/682) and pull request [#1012](https://github.com/lme4/lme4/pull/1012)). - -If all group-level terms of a binomial (logit link) or Poisson (log link) submodel are group-level intercepts (`(1 | )`), **projpred** fits the submodel with its own Laplace approximation, which is faster than `lme4::glmer()`. -Submodels with other group-level terms (for example, group-level slopes) are fitted with `lme4::glmer()`, and then the projection can take long, in particular for Poisson models. -According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in that case. - -Finally, the [Poisson example of the latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html#example-poisson-distribution) shows that the latent projection can be beneficial for non-multilevel models with a (non-Gaussian) family that the traditional projection also supports. +We recommend `cv_varsel()` instead of `varsel()` and `cv_varsel()` should be used for final results anyway. Similarly, `cv_varsel()` with `validate_search = FALSE` is more prone to overfitting than `cv_varsel()` with `validate_search = TRUE`. ### Issues with the augmented-data projection @@ -496,8 +483,6 @@ For multilevel models, the augmented-data projection may not work properly and g By suboptimal results, we mean that the relevance of the group-level terms can be underestimated. According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in such cases. -If the only group-level term of a `brms::cumulative()` submodel is a group-level intercept (`(1 | )`), **projpred** fits the submodel with its own Laplace approximation, which is faster than `ordinal::clmm()`. - ### Speed {#speed} For Gaussian models with the identity link and without multilevel or additive terms, and for the latent projection without such terms, **projpred** computes the forward search and the LOO CV with validated search from cross-products of the model matrix. From c8c6cc95d0ce854a8895914a4754050838bded4a Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 19:01:15 +0200 Subject: [PATCH 36/40] Restore the warnings of init_refmodel() for multilevel binomial and Poisson models --- R/refmodel.R | 16 ++++++++++------ tests/testthat/test_refmodel.R | 23 ----------------------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/R/refmodel.R b/R/refmodel.R index 6f50dbdc6..5fceb1f52 100644 --- a/R/refmodel.R +++ b/R/refmodel.R @@ -1270,13 +1270,17 @@ init_refmodel <- function(object, data, formula, family, ref_predfun = NULL, "properly. The latent projection may be a remedy. See section ", "\"Troubleshooting\" of the main vignette for more information." ) - } else if (family$family %in% c("binomial", "poisson") && - utils::packageVersion("lme4") < "2.1.1") { + } else if (family$family == "binomial") { warning( - "With lme4 < 2.1-1, the group-level standard deviations of ", - "multilevel binomial and Poisson submodels can be wrong. We ", - "recommend lme4 >= 2.1-1. See section \"Troubleshooting\" of the ", - "main vignette for more information." + "For multilevel binomial models, the traditional projection may not ", + "work properly. The latent projection may be a remedy. See section ", + "\"Troubleshooting\" of the main vignette for more information." + ) + } else if (family$family == "poisson") { + warning( + "For multilevel Poisson models, the traditional projection may take ", + "very long. The latent projection may be a remedy. See section ", + "\"Troubleshooting\" of the main vignette for more information." ) } } diff --git a/tests/testthat/test_refmodel.R b/tests/testthat/test_refmodel.R index 7884b247f..9e0180f18 100644 --- a/tests/testthat/test_refmodel.R +++ b/tests/testthat/test_refmodel.R @@ -641,26 +641,3 @@ test_that(paste( proj_linpred(prj_nt, newdata = dat_nt[1:5, c("x1", "x2", "y")]) ) }) - -test_that(paste( - "init_refmodel() warns about lme4 < 2.1-1 for multilevel binomial and", - "Poisson models exactly if lme4 < 2.1-1" -), { - tstsetups <- grep("\\.glmm\\.(brnll|binom|poiss)\\.", names(fits), - value = TRUE) - skip_if(length(tstsetups) == 0) - withr::local_options(list(projpred.warn_instable_projections = NULL)) - lme4_old <- utils::packageVersion("lme4") < "2.1.1" - for (tstsetup in tstsetups) { - warns <- character() - suppressMessages(withCallingHandlers( - get_refmodel(fits[[tstsetup]]), - warning = function(w) { - warns <<- c(warns, conditionMessage(w)) - invokeRestart("muffleWarning") - } - )) - expect_identical(any(grepl("With lme4 < 2.1-1", warns, fixed = TRUE)), - lme4_old, info = tstsetup) - } -}) From 49755b6f3bc422ec7b13de39a0e52de1bbdcfe95 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Tue, 17 Nov 2026 23:39:31 +0200 Subject: [PATCH 37/40] Limit the augmented-data troubleshooting text to the cumulative and categorical families --- vignettes/projpred.Rmd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 672530314..97272e6d7 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -479,7 +479,8 @@ We recommend `cv_varsel()` instead of `varsel()` and `cv_varsel()` should be use ### Issues with the augmented-data projection -For multilevel models, the augmented-data projection may not work properly and give suboptimal results, see [#353](https://github.com/stan-dev/projpred/pull/353). +For multilevel models with the `brms::cumulative()` or the `brms::categorical()` family (or `rstanarm::stan_polr()` fits), the augmented-data projection may not work properly and give suboptimal results, see [#353](https://github.com/stan-dev/projpred/pull/353). +For the `binomial()` and the `brms::bernoulli()` family, the augmented-data projection uses the same `lme4::glmer()` fits as the traditional projection. By suboptimal results, we mean that the relevance of the group-level terms can be underestimated. According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in such cases. From 141f2435dde2a634d004ffc3fc202f46ea867868 Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Wed, 18 Nov 2026 09:22:36 +0200 Subject: [PATCH 38/40] Add the PR number to the NEWS item --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 26f480f73..b34a9ca78 100644 --- a/NEWS.md +++ b/NEWS.md @@ -46,7 +46,7 @@ If you read this from a place other than Date: Wed, 18 Nov 2026 13:09:14 +0200 Subject: [PATCH 39/40] Explain the multilevel issue of the binomial and augmented-data projections in the troubleshooting section --- vignettes/projpred.Rmd | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 97272e6d7..6d0a5584b 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -477,12 +477,17 @@ In this case, the latent-space approach by @catalina_latent_2021 might help, see If `varsel()` is used, the lack of a CV in `varsel()` may lead to overconfident and overfitted results. We recommend `cv_varsel()` instead of `varsel()` and `cv_varsel()` should be used for final results anyway. Similarly, `cv_varsel()` with `validate_search = FALSE` is more prone to overfitting than `cv_varsel()` with `validate_search = TRUE`. -### Issues with the augmented-data projection - -For multilevel models with the `brms::cumulative()` or the `brms::categorical()` family (or `rstanarm::stan_polr()` fits), the augmented-data projection may not work properly and give suboptimal results, see [#353](https://github.com/stan-dev/projpred/pull/353). -For the `binomial()` and the `brms::bernoulli()` family, the augmented-data projection uses the same `lme4::glmer()` fits as the traditional projection. -By suboptimal results, we mean that the relevance of the group-level terms can be underestimated. -According to the simulation-based case study from [#353](https://github.com/stan-dev/projpred/pull/353), the latent projection might help in such cases. +### Issues with multilevel models + +In multilevel models, the projected group-level standard deviations can be near zero for the `binomial()` and `brms::bernoulli()` families (traditional and augmented-data projection). +The same applies to the augmented-data projection of the `brms::cumulative()` and `brms::categorical()` families and of `rstanarm::stan_polr()` fits. +In that case, the search can underestimate the relevance of the group-level terms, see [#353](https://github.com/stan-dev/projpred/pull/353). +**projpred** estimates these standard deviations by (approximate) maximum marginal likelihood, which assumes binomial or multinomial noise around the fitted values. +The projection fits the submodels to the reference model's predictions, which have no noise, so the data have less noise than the model assumes. +The group-level standard deviations are likely to be near zero if the groups have few observations or the observations have few trials. +The augmented-data projection has one trial per observation. +The latent projection might help because its Gaussian submodels estimate the residual standard deviation (see the [latent-projection vignette](https://mc-stan.org/projpred/articles/latent.html)). +The latent projection is not available for the `brms::categorical()` family. ### Speed {#speed} From 7ec6566dcaf1aa29402891d3ccaa90762e4f689d Mon Sep 17 00:00:00 2001 From: Aki Vehtari Date: Wed, 18 Nov 2026 14:27:23 +0200 Subject: [PATCH 40/40] Shorten the parallelization paragraph of the main vignette --- vignettes/projpred.Rmd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vignettes/projpred.Rmd b/vignettes/projpred.Rmd index 6d0a5584b..123921e30 100755 --- a/vignettes/projpred.Rmd +++ b/vignettes/projpred.Rmd @@ -141,7 +141,8 @@ This run takes only a few seconds. The code below also shows how **projpred**'s CV can be parallelized with the **future** package (we disable the parallelization for the vignette build). The LOO folds are then split among the parallel workers. For a model as small as this one, the parallelization overhead can be larger than the runtime gain. -We recommend the workers of the **future.mirai** package: in our experiments, the first parallel call was faster with them than with `future::multisession` (see section "Parallelization" in the general package documentation available [online](https://mc-stan.org/projpred/reference/projpred-package.html) or by typing `` ?`projpred-package` ``). +We use the workers of the **future.mirai** package because the first parallel call was faster with them than with `future::multisession`. +For more details, see section "Parallelization" in the package documentation ([online](https://mc-stan.org/projpred/reference/projpred-package.html)) or type `` ?`projpred-package` ``. If PSIS-LOO CV is not reliable (for example, if many Pareto $\hat{k}$ values are high), $K$-fold CV (`cv_method = "kfold"`) is an alternative. Then `run_cvfun()` can run the $K$ reference model refits once, so that several `cv_varsel()` calls can use them (see argument `cvfits` of `cv_varsel()`).