update submodule.

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "paper"]
 	path = paper
 	url = git@github.com:chainsawriot/measure.git
+[submodule "overleaf"]
+	path = overleaf
+	url = https://git.overleaf.com/62a956eb9b9254783cc84c82
+[submodule "misclassificationmodels"]
+	path = misclassificationmodels
+	url = https://github.com/chainsawriot/misclassificationmodels.git
+[submodule "presentation"]
+	path = presentation
+	url = https://git.overleaf.com/646be7922a7fb19bcb461593

charts/.gitignore

@@ -0,0 +1,9 @@
+*.aux
+*.svg
+*.pdf
+*.png
+*.aux
+*.fdb_latexmk
+*.log
+*.fls
+auto/*

charts/Makefile

@@ -0,0 +1,29 @@
+#!/usr/bin/make
+
+all: $(patsubst %.tex,%.svg,$(wildcard *.tex)) $(patsubst %.tex,%.png,$(wildcard *.tex)) 
+
+%.png: %.pdf
+	convert -density 300 -transparent white $< $@
+
+%.svg: %.pdf
+	/usr/bin/inkscape $< --export-plain-svg --export-type=svg --export-filename=$@
+
+%.pdf: %.tex 
+	latexmk -f -pdf $<
+
+clean: 
+	latexmk -C *.tex
+	rm -f *.tmp
+	rm -f vc
+	rm *.svg
+
+
+viewpdf: all
+	evince *.pdf
+
+vc:
+	vc-git
+
+pdf: all
+
+.PHONY: clean all

charts/example_1_dag.tex

@@ -0,0 +1,46 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows, }
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+\begin{tikzpicture}
+
+  \node[observed] (y) {$Y$};
+  \node[unobserved, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+%  \node[unobserved, above=of w] (k) {$K$};
+  \node[observed,right=of x] (z) {$Z$};
+%   \node[residual,below=of y] (e) {$\varepsilon$};
+%  \node[residual,below=of w] (xi) {$\xi$};
+
+  \draw[->] (z) to (y);
+  \draw[->] (z) -- (x);
+  \draw[->] (x) -- (y);
+  \draw[->] (x) -- (w);
+%  \draw[->] (y) -- (w);
+%  \draw[->] (x) -- (xi);
+ % \draw[->] (w) -- (xi);
+\end{tikzpicture}
+\end{document}
+

charts/example_2_dag.tex

@@ -0,0 +1,47 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+\begin{tikzpicture}
+  \node[observed] (y) {$Y$};
+  \node[unobserved, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+  \node[observed,right=of x] (z) {$Z$};
+
+  \draw[->] (z) to (y);
+  \draw[->] (z) -- (x);
+  \draw[->] (x) -- (y);
+  \draw[->] (x) -- (w);
+
+  \draw[->] (x) to (y);
+
+  \draw[->] (y) -- (w);
+  \draw[->] (y) -- (w);
+
+
+\end{tikzpicture}
+\end{document}
+

charts/example_3_dag.tex

@@ -0,0 +1,42 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+
+\begin{tikzpicture}
+  \node[unobserved] (y) {$Y$};
+
+  \node[observed, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+%  \node[unobserved, above=of w] (k) {$K$};
+  \node[observed,right=of x] (z) {$Z$};
+%   \node[residual,below=of y] (e) {$\varepsilon$};
+  % \node[residual,below=of w] (xi) {$\xi$};
+  \draw[->] (z) to (y);
+  \draw[->] (x) -- (y);
+  \draw[->] (y) -- (w);
+\end{tikzpicture}
+\end{document}
+

charts/example_4_dag.tex

@@ -0,0 +1,47 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+
+\begin{tikzpicture}
+  \node[unobserved] (y) {$Y$};
+
+  \node[observed={white}{gray!40}, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+%  \node[unobserved, above=of w] (k) {$K$};
+  \node[observed,right=of x] (z) {$Z$};
+%   \node[residual,below=of y] (e) {$\varepsilon$};
+%   \node[residual,below=of w] (xi) {$\xi$};
+  \draw[->] (x) -- (y);
+%  \draw[->] (x) -- (w);
+  \draw[->] (y) -- (w);
+%  \draw[->] (k) -- (w);
+  \draw[->] (z) -- (y);
+%  \draw[->] (z) -- (k);
+%  \draw[->] (y) -- (xi);
+%  \draw[->] (w) -- (xi);
+\end{tikzpicture}
+\end{document}
+

charts/example_legend.tex

@@ -0,0 +1,35 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+
+\begin{tikzpicture}
+  \matrix [draw, below, font=\small, align=center, column sep=2\pgflinewidth, inner sep=0.4em, outer sep=0em, nodes={align=center, anchor=center}] at (current bounding box.south){
+    \node[observed,label=right:observed] {}; \\
+    \node[unobserved,label=right:automatically classified]{}; \\
+    \node[residual,label=right:error term]{}; \\
+  };
+\end{tikzpicture}
+\end{document}
+

civil_comments/03_prob_not_pred.R

@@ -0,0 +1,21 @@
+source('load_perspective_data.R')
+source("../simulations/RemembR/R/RemembeR.R")
+library(xtable)
+change.remember.file("prob_not_pred.RDS")
+### to respond to the reviewer show what happens if we don't recode the predictions.
+
+non_recoded_dv <- lm(toxicity_prob ~ likes * race_disclosed, data=df)
+remember(coef(non_recoded_dv), "coef_dv")
+remember(diag(vcov(non_recoded_dv)), "se_dv")
+remember(xtable(non_recoded_dv),'dv_xtable')
+
+non_recoded_iv <- glm(race_disclosed ~ likes * toxicity_prob, data=df, family='binomial')
+remember(coef(non_recoded_iv), "coef_iv")
+remember(diag(vcov(non_recoded_iv)), "se_iv")
+remember(xtable(non_recoded_iv),'iv_xtable')
+
+remember(extract(non_recoded_iv,include.aic=F,include.bic=F,include.nobs=F,include.deviance=F,include.loglik=F),'non_recoded_iv')
+remember(extract(non_recoded_dv,include.rsquared=F,include.adjrs=F,include.nobs=F),'non_recoded_dv')
+tr <- texreg(list(r$non_recoded_iv, r$non_recoded_dv),custom.model.names=c("Example 1","Example 2"),custom.coef.map=list("(Intercept)"="Intercept","race_disclosedTRUE"="Identity Disclosure","toxicity_prob"="Toxicity Score","likes"="Likes","likes:race_disclosedTRUE"="Likes:Identity Disclosure","likes:toxicity_prob"="Likes:Toxicity Score"),single.row=T,dcolumn=T)
+print(tr)
+remember(tr, 'texregobj')

civil_comments/Makefile

@@ -1,4 +1,4 @@
-qall: iv_perspective_example.RDS dv_perspective_example.RDS
+all: iv_perspective_example.RDS dv_perspective_example.RDS
 
 srun_1core=srun -A comdata -p compute-bigmem --mem=4G --time=12:00:00 -c 1 --pty /usr/bin/bash -l
 srun=srun -A comdata -p compute-bigmem --mem=4G --time=12:00:00 --pty /usr/bin/bash -l

irr/irr.R

@@ -0,0 +1,43 @@
+require(tibble)
+require(purrr)
+
+.emulate_coding <- function(ground_truth, Q = 1) {
+    if (runif(1) > Q) {
+        return(sample(c(1,0), 1))
+    } else {
+        return(ground_truth)
+    }
+}
+
+##irr::kripp.alpha(matrix(c(obs_x, obs_x2), nrow = 2, byrow = TRUE), method = "nominal")
+### Which is very close to
+## cor(obs_x, obs_x2)
+
+.sim <- function(N = 100, P = 0.5, Q = 0.8) {
+    real_x <- rbinom(N, 1, P)
+    obs_x <- purrr::map_dbl(real_x, .emulate_coding, Q = Q)
+### then learn w from obs_x and k
+    obs_x2 <- purrr::map_dbl(real_x, .emulate_coding, Q = Q)
+    ra <- sum(diag(table(obs_x, obs_x2))) / N ## raw agreement
+    rr <- cor(obs_x, obs_x2)
+    irr <- irr::kripp.alpha(matrix(c(obs_x, obs_x2), nrow = 2, byrow = TRUE), method = "nominal")$value
+    return(data.frame(N, P, Q, ra, rr, irr))
+}
+
+N <- c(50, 100, 300)
+P <- c(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)
+Q <- c(0.5, 0.6, 0.7, 0.8, 0.9, 1)
+conditions <- tibble::tibble(expand.grid(N, P, Q))
+colnames(conditions) <- c("N", "P", "Q")
+res <- list()
+
+for (i in seq_len(nrow(conditions))) {
+    print(i)
+    res[[i]] <- purrr::map_dfr(rep(NA, 100), ~ .sim(conditions$N[i], conditions$P[i], conditions$Q[i]))
+}
+
+conditions$res <- res
+
+require(dplyr)
+
+conditions %>% mutate(mra = purrr::map_dbl(res, ~mean(.$ra, na.rm = TRUE)), mrr = purrr::map_dbl(res, ~mean(.$rr, na.rm = TRUE)), mirr = purrr::map_dbl(res, ~mean(.$irr, na.rm = TRUE))) %>% lm(mirr~0+P+poly(Q, 2), data =.) %>% summary

irr/loco_loa.R

@@ -0,0 +1,81 @@
+.emulate_coding <- function(ground_truth, Q = 1) {
+    if (runif(1) > Q) {
+        return(sample(c(0, 1), size = 1, replace = TRUE))
+    } else {
+        return(ground_truth)
+    }
+}
+
+distort_gt <- function(x, Q = NULL) {
+    return(purrr::map_dbl(x, .emulate_coding, Q = Q))
+}
+
+N <- c(1000, 3600, 14400)
+m <- c(75, 150, 300)
+
+B0 <- c(0, 0.1, 0.3)
+Bxy <- c(0.1, 0.2, 0.5)
+
+Q <- c(.6, .8, .9)
+
+conditions <- expand.grid(N, m, B0, Bxy, Q)
+
+colnames(conditions) <- c("N", "m", "B0", "Bxy", "Q")
+
+logistic <- function(x) {1/(1+exp(-1*x))}
+
+.step <- function(i, Bxy, B0, Q, N, m) {
+    x <- rbinom(N, 1, 0.5)
+    y <-  Bxy * x + rnorm(N, 0, .5) + B0
+
+    dx <- as.numeric(distort_gt(x, Q = Q))
+
+    randomidx <- sample(seq(N), m)
+    
+    coder1x <- distort_gt(x[randomidx], Q = Q)
+    coder2x <- distort_gt(x[randomidx], Q = Q)
+    coding_data <- matrix(c(as.numeric(coder1x), as.numeric(coder2x)), nrow = 2, byrow = TRUE)
+    alpha <- irr::kripp.alpha(coding_data, method = "nominal")
+    estimated_q <- alpha$value^(1/2)
+    estimated_q2 <- alpha$value
+    
+    res <- data.frame(x = as.factor(x), y = y, dx = as.factor(dx))
+    small_y <- y[randomidx]
+    small_x <- x[randomidx]
+    naive_mod <- glm(y~dx, data = res, x = TRUE, y = TRUE)
+    real_mod <- glm(y~x, data = res, x = TRUE, y = TRUE)
+    m1 <- glm(small_y~coder1x)
+    m2 <- glm(small_y~coder2x)
+    m3 <- glm(small_y~small_x)
+    correct_only_idx <- coder1x == coder2x
+    m4 <- glm(small_y[correct_only_idx] ~ small_x[correct_only_idx])
+    lab_only_gt <- coef(m3)[2]
+    lab_only_avg <- mean(coef(m1)[2], coef(m2)[2])
+    lab_only_correct_only <- coef(m4)[2]    
+    return(tibble::tibble(N, m, Q, Bxy, B0, estimated_q, naive_Bxy = as.numeric(coef(naive_mod)[2]), real_Bxy = as.numeric(coef(real_mod)[2]), lab_only_gt= lab_only_gt, lab_only_avg = lab_only_avg, lab_only_correct_only = lab_only_correct_only))
+}
+
+## res <- list()
+
+## for (i in seq(nrow(conditions))) {
+##     message(i)
+##     res[[i]] <- purrr::map_dfr(1:100, ~.step(., conditions$Bxy[i], conditions$B0[i], conditions$Q[i], conditions$N[i], conditions$m[i]))
+## }
+
+require(furrr)
+plan(multisession)
+
+.run <- function(i, conditions) {
+    purrr::map_dfr(1:100, ~.step(., conditions$Bxy[i], conditions$B0[i], conditions$Q[i], conditions$N[i], conditions$m[i]))
+}
+
+res <- future_map(seq(nrow(conditions)), .run, conditions = conditions, .progress = TRUE)
+
+##saveRDS(res, "rubin_res.RDS")
+
+conditions <- tibble::as_tibble(conditions)
+conditions$res <- res
+
+require(tidyverse)
+
+conditions %>% mutate(loco_median = purrr::map_dbl(res, ~median(.$lab_only_correct_only)), loco_p025 = purrr::map_dbl(res, ~quantile(.$lab_only_correct_only, probs = 0.025)), loco_p975 = purrr::map_dbl(res, ~quantile(.$lab_only_correct_only, probs = 0.975))) %>% mutate(loa_median = purrr::map_dbl(res, ~median(.$lab_only_avg)), loa_p025 = purrr::map_dbl(res, ~quantile(.$lab_only_avg, probs = 0.025)), loa_p975 = purrr::map_dbl(res, ~quantile(.$lab_only_avg, probs = 0.975))) %>% filter(B0 == 0.1 & Bxy == 0.5) %>% select(N, m, Q, starts_with("loco"), starts_with("loa")) %>% pivot_longer(cols = loco_median:loa_p975, names_to = c("type", "tile"),names_pattern = "(.*)_(.*)", values_to = "value") %>% pivot_wider(names_from = "tile") %>% ggplot(aes(x = Q, y = median, ymin = p025, ymax = p975, fill = type, col = type))  + geom_line() + geom_ribbon(alpha = 0.2) + facet_grid(N~m) + geom_hline(yintercept = .5, linetype = 2, col = "grey")

irr/simex_sim.R

@@ -0,0 +1,56 @@
+##install.packages(c("purrr", "simex", "irr"))
+
+.emulate_coding <- function(ground_truth, Q = 1) {
+    if (runif(1) > Q) {
+        return(sample(c(0, 1), size = 1, replace = TRUE))
+    } else {
+        return(ground_truth)
+    }
+}
+
+distort_gt <- function(x, Q = NULL) {
+    return(purrr::map_dbl(x, .emulate_coding, Q = Q))
+}
+
+N <- c(1000, 3600, 14400)
+m <- c(75, 150, 300)
+
+B0 <- c(0, 0.1, 0.3)
+Bxy <- c(0.1, 0.2, 0.5)
+
+Q <- c(.6, .8, .9)
+
+conditions <- expand.grid(N, m, B0, Bxy, Q)
+
+logistic <- function(x) {1/(1+exp(-1*x))}
+
+.step <- function(Bxy, B0, Q, N, m) {
+    x <- rbinom(N, 1, 0.5)
+    y <-  Bxy * x + rnorm(N, 0, .5) + B0
+
+    dx <- as.numeric(distort_gt(x, Q = Q))
+
+    randomx <- sample(x, m)
+    coder1x <- distort_gt(randomx, Q = Q)
+    coder2x <- distort_gt(randomx, Q = Q)
+    coding_data <- matrix(c(as.numeric(coder1x), as.numeric(coder2x)), nrow = 2, byrow = TRUE)
+    alpha <- irr::kripp.alpha(coding_data, method = "nominal")
+    estimated_q <- alpha$value^(1/2)
+    estimated_q2 <- alpha$value
+
+    res <- data.frame(x = as.factor(x), y = y, dx = as.factor(dx))
+
+    naive_mod <- glm(y~dx, data = res, x = TRUE, y = TRUE)
+    real_mod <- glm(y~x, data = res, x = TRUE, y = TRUE)
+
+    px <- matrix(c(estimated_q, 1-estimated_q, 1-estimated_q, estimated_q), nrow = 2)
+    colnames(px) <- levels(res$dx)
+    corrected_mod <- simex::mcsimex(naive_mod, SIMEXvariable = "dx", mc.matrix = px, jackknife.estimation = FALSE, B = 300)
+    px2 <- matrix(c(estimated_q2, 1-estimated_q2, 1-estimated_q2, estimated_q2), nrow = 2)
+    colnames(px2) <- levels(res$dx)
+    corrected_mod2 <- simex::mcsimex(naive_mod, SIMEXvariable = "dx", mc.matrix = px2, jackknife.estimation = FALSE, B = 300)
+
+    return(tibble::tibble(N, m, Q, Bxy, B0, estimated_q, naive_Bxy = as.numeric(coef(naive_mod)[2]), real_Bxy = as.numeric(coef(real_mod)[2]), corrected_Bxy = coef(corrected_mod)[2], corrected_Bxy2 = coef(corrected_mod2)[2]))
+}
+
+## res <- .step(0.2, 0, 0.8, N = 1000, m = 100)

simulations/03_depvar.R

@@ -73,8 +73,8 @@ parser <- add_argument(parser, "--y_explained_variance", help='what proportion o
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.72)
 ## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
 ## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
-parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.01)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
-parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.01)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
 parser <- add_argument(parser, "--Bzx", help='coeffficient of z on x', default=-0.5)
 parser <- add_argument(parser, "--B0", help='Base rate of y', default=0.5)
 parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")

simulations/03_indep_differential_nonnorm.R

@@ -0,0 +1,185 @@
+### EXAMPLE 1: demonstrates how measurement error can lead to a type sign error in a covariate
+### What kind of data invalidates fong + tyler?
+### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
+### Even when you include the proxy variable in the regression.
+### But with some ground truth and multiple imputation, you can fix it.
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+library(predictionError)
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+setDTthreads(40)
+
+source("simulation_base.R")
+
+## SETUP:
+### we want to estimate x -> y; x is MAR
+### we have x -> k; k -> w; x -> w is used to predict x via the model w.
+### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
+### The labels x are binary, but the model provides a continuous predictor
+
+### simulation:
+#### how much power do we get from the model in the first place? (sweeping N and m)
+#### 
+
+## one way to do it is by adding correlation to x.obs and y that isn't in w.
+## in other words, the model is missing an important feature of x.obs that's related to y.
+simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, y_bias=-0.8,z_bias=0,Px=0.5,accuracy_imbalance_difference=0.3,sd_y_mixin=1){
+    set.seed(seed)
+    # make w and y dependent
+    z <- rnorm(N,sd=0.5)
+    x <- rbinom(N, 1, plogis(Bzx * z + qlogis(Px)))
+    ## following Fong + Tyler: mix y with a Bernoulli(0.15) × |N (0, 20)| to make a skewed non-normal distribution
+    y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
+    y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
+    y <- Bzy * z + Bxy * x + y.epsilon + rbinom(N,1,0.15) * rnorm(N,0,sd_y_mixin)
+    
+    df <- data.table(x=x,y=y,z=z)
+
+    if(m < N){
+        df <- df[sample(nrow(df), m), x.obs := x]
+    } else {
+        df <- df[, x.obs := x]
+    }
+
+    ## probablity of an error is correlated with y
+    ## pz <- mean(z)
+    ## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
+
+    ## # this works because of conditional probability
+    ## accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+    ## accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
+
+    ## z0x0 <- df[(z==0) & (x==0)]$x
+    ## z0x1 <- df[(z==0) & (x==1)]$x
+    ## z1x0 <- df[(z==1) & (x==0)]$x
+    ## z1x1 <- df[(z==1) & (x==1)]$x
+
+    ## yz0x0 <- df[(z==0) & (x==0)]$y
+    ## yz0x1 <- df[(z==0) & (x==1)]$y
+    ## yz1x0 <- df[(z==1) & (x==0)]$y
+    ## yz1x1 <- df[(z==1) & (x==1)]$y
+
+    ## nz0x0 <- nrow(df[(z==0) & (x==0)])
+    ## nz0x1 <- nrow(df[(z==0) & (x==1)])
+    ## nz1x0 <- nrow(df[(z==1) & (x==0)])
+    ## nz1x1 <- nrow(df[(z==1) & (x==1)])
+
+    ## yz1 <- df[z==1]$y 
+    ## yz1 <- df[z==1]$y 
+
+    ## # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+    ## acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
+    ## acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
+    ## acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
+    ## acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
+
+    ## w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
+    ## w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
+    ## w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
+    ## w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
+
+    ## ##perrorz0 <- w0z0*(pyz0)
+    ## ##perrorz1 <- w0z1*(pyz1)
+
+    ## w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
+    ## w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
+    ## w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
+    ## w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
+
+    ## df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
+    ## df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]    
+    ## df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]    
+    ## df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]    
+
+    ## df[,w_pred:=as.integer(w > 0.5)]
+    ## print(mean(df[z==0]$x == df[z==0]$w_pred))
+    ## print(mean(df[z==1]$x == df[z==1]$w_pred))
+    ## print(mean(df$w_pred == df$x))
+
+
+
+    resids <- resid(lm(y~x + z))
+    odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(resids[x==1],log.p=T),log.p=T) + z_bias * qlogis(pnorm(z[x==1],sd(z),log.p=T),log.p=T)
+    odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(resids[x==0],log.p=T),log.p=T) + z_bias * qlogis(pnorm(z[x==0],sd(z),log.p=T),log.p=T)
+
+    ## acc.x0 <- p.correct[df[,x==0]]
+    ## acc.x1 <- p.correct[df[,x==1]]
+
+    df[x==0,w:=plogis(rlogis(.N,odds.x0))]
+    df[x==1,w:=plogis(rlogis(.N,odds.x1))]
+
+    print(prediction_accuracy)
+    print(resids[is.na(df$w)])
+    print(odds.x0[is.na(df$w)])
+    print(odds.x1[is.na(df$w)])
+
+    df[,w_pred := as.integer(w > 0.5)]
+
+
+    print(mean(df$w_pred == df$x))
+    print(mean(df[y>=0]$w_pred == df[y>=0]$x))
+    print(mean(df[y<=0]$w_pred == df[y<=0]$x))
+    return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=51, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.75)
+parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
+parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*z*x")
+parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-0.5)
+parser <- add_argument(parser, "--z_bias", help='coefficient of z on the probability a classification is correct', default=0)
+parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
+parser <- add_argument(parser, "--Px", help='base rate of x', default=0.5)
+parser <- add_argument(parser, "--confint_method", help='method for approximating confidence intervals', default='quad')
+parser <- add_argument(parser, "--sd_y_mixin", help='varience of the non-normal part of Y', default=10)
+args <- parse_args(parser)
+
+B0 <- 0
+Px <- args$Px
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+Bzx <- args$Bzx
+
+if(args$m < args$N){
+
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, y_bias=args$y_bias, sd_y_mixin=args$sd_y_mixin)
+
+    ## df.pc <- df[,.(x,y,z,w_pred,w)]
+    ##                                     #    df.pc <- df.pc[,err:=x-w_pred]
+    ## pc.df <- pc(suffStat=list(C=cor(df.pc),n=nrow(df.pc)),indepTest=gaussCItest,labels=names(df.pc),alpha=0.05)
+    ## plot(pc.df)
+
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, 'Bzx'=args$Bzx, 'Bzy'=Bzy, 'Px'=Px, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'y_bias'=args$y_bias,'outcome_formula'=args$outcome_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, confint_method=args$confint_method, error='', 'sd_y_mixin'=args$sd_y_mixin)
+
+    outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula),confint_method=args$confint_method)
+    
+   
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }
+
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

simulations/Makefile

@@ -8,7 +8,7 @@ explained_variances=[0.1]
 
 all:main supplement
 main:remembr.RDS 
-supplement:robustness_1.RDS robustness_1_dv.RDS robustness_2.RDS robustness_2_dv.RDS robustness_3.RDS robustness_3_dv.RDS robustness_3_proflik.RDS robustness_3_dv_proflik.RDS robustness_4.RDS robustness_4_dv.RDS 
+supplement:robustness_1.RDS robustness_1_dv.RDS robustness_2.RDS robustness_2_dv.RDS robustness_3.RDS robustness_3_dv.RDS robustness_3_proflik.RDS robustness_3_dv_proflik.RDS robustness_4.RDS robustness_4_dv.RDS robustness_5.RDS robustness_5_dv.RDS robustness_6.feather
 
 srun=sbatch --wait --verbose run_job.sbatch
 
@@ -16,7 +16,7 @@ srun=sbatch --wait --verbose run_job.sbatch
 joblists:example_1_jobs example_2_jobs example_3_jobs
 
 # test_true_z_jobs: test_true_z.R simulation_base.R
-# 	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
+# 	sbatch --wait --verbose run_job.sbatch  ./grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
 
 # test_true_z.feather: test_true_z_jobs 
 # 	rm -f test_true_z.feather
@@ -25,48 +25,46 @@ joblists:example_1_jobs example_2_jobs example_3_jobs
 
 
 example_1_jobs: 01_two_covariates.R simulation_base.R grid_sweep.py pl_methods.R
-	${srun} grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[1]}' --outfile example_1_jobs
+	${srun} ./grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[1]}' --outfile example_1_jobs
 
 example_1.feather: example_1_jobs 
 	rm -f example_1.feather
 	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_1_jobs
 	sbatch --wait --verbose --array=1001-2000 run_simulation.sbatch 0 example_1_jobs
 	sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_1_jobs
-	sbatch --wait --verbose --array=3001-4000  run_simulation.sbatch 0 example_1_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
-	sbatch --wait --verbose --array=4001-$(shell cat example_1_jobs | wc -l)  run_simulation.sbatch 0 example_1_jobs
 
 example_2_jobs: 02_indep_differential.R simulation_base.R grid_sweep.py pl_methods.R
-	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+z+x"]}' --outfile example_2_jobs
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+z+x"]}' --outfile example_2_jobs
 
 example_2.feather: example_2_jobs 
 	rm -f example_2.feather
 	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_2_jobs
 	sbatch --wait --verbose --array=1001-2000 run_simulation.sbatch 0 example_2_jobs
 	sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_2_jobs
-	sbatch --wait --verbose --array=3001-4000  run_simulation.sbatch 0 example_2_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
-	sbatch --wait --verbose --array=4001-$(shell cat example_2_jobs | wc -l)
+
 
 
 # example_2_B_jobs: example_2_B.R
-# 	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
+# 	sbatch --wait --verbose run_job.sbatch python3 ./grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
 
 # example_2_B.feather: example_2_B_jobs
 # 	rm -f example_2_B.feather
 # 	sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
 
 example_3_jobs: 03_depvar.R simulation_base.R grid_sweep.py pl_methods.R
-	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "Bxy":[0.7],"Bzy":[-0.7],"Bzx":[1],"seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "Bxy":[0.7],"Bzy":[-0.7],"Bzx":[1],"seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
 
 example_3.feather: example_3_jobs
 	rm -f example_3.feather	
 	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_3_jobs
 	sbatch --wait --verbose --array=1001-2000 run_simulation.sbatch 0 example_3_jobs
 	 sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_3_jobs
-	sbatch --wait --verbose --array=3001-4000  run_simulation.sbatch 0 example_3_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs
-	sbatch --wait --verbose --array=4001-$(shell cat example_3_jobs | wc -l)  run_simulation.sbatch 0 example_3_jobs
 
 example_4_jobs: 04_depvar_differential.R simulation_base.R grid_sweep.py pl_methods.R
-	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"Bxy":[0.7],"Bzy":[-0.7],"Bzx":[1], "m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[-0.5], "prediction_accuracy":[0.73]}' --outfile example_4_jobs
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"Bxy":[0.7],"Bzy":[-0.7],"Bzx":[1], "m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[-0.5], "prediction_accuracy":[0.73]}' --outfile example_4_jobs
 
 example_4.feather: example_4_jobs
 	rm -f example_4.feather
@@ -74,9 +72,7 @@ example_4.feather: example_4_jobs
 	sbatch --wait --verbose --array=1001-2000  run_simulation.sbatch 0 example_4_jobs
 	sbatch --wait --verbose --array=2001-3001 run_simulation.sbatch 0 example_4_jobs
 	sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_4_jobs
-	sbatch --wait --verbose --array=3001-4000  run_simulation.sbatch 0 example_4_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_4_jobs | wc -l)  run_simulation.sbatch 0 example_4_jobs
-	sbatch --wait --verbose --array=4001-$(shell cat example_4_jobs | wc -l)  run_simulation.sbatch 0 example_4_jobs
-
 
 
 remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R summarize_estimator.R
@@ -92,22 +88,21 @@ STEP=1000
 ONE=1
 
 robustness_Ns=[1000,5000]
-robustness_robustness_ms=[100,200]
+robustness_ms=[100,200]
 
 #in robustness 1 / example 2 misclassification is correlated with Y.
 robustness_1_jobs_p1: 02_indep_differential.R simulation_base.R grid_sweep.py
-	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":[1000],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3,0],"Bxy":[0.3],"Bzx":[1,0], "outcome_formula":["y~x+z"], "z_bias":[0, 0.5], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs_p1
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":[1000],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3,0],"Bxy":[0.3],"Bzx":[1,0], "outcome_formula":["y~x+z"], "z_bias":[0, 0.5], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs_p1
 
 robustness_1_jobs_p2: 02_indep_differential.R simulation_base.R grid_sweep.py
-	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":[5000],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3,0],"Bxy":[0.3],"Bzx":[1,0], "outcome_formula":["y~x+z"], "z_bias":[0, 0.5], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs_p2
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":[5000],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3,0],"Bxy":[0.3],"Bzx":[1,0], "outcome_formula":["y~x+z"], "z_bias":[0, 0.5], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs_p2
 
 robustness_1.feather: robustness_1_jobs_p1 robustness_1_jobs_p2
 	rm -f $@
 	$(eval END_1!=cat robustness_1_jobs_p1 | wc -l)
-	$(eval ITEROBUSTNESS_MS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
 	$(eval END_2!=cat robustness_1_jobs_p2 | wc -l)
-	$(eval ITEROBUSTNESS_MS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
-
 	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_1_jobs_p1;)
 	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_1_jobs_p2;)
 
@@ -117,10 +112,10 @@ robustness_1.RDS: robustness_1.feather summarize_estimator.R
 
 # when Bzy is 0 and zbias is not zero, we have the case where P(W|Y,X,Z) has an omitted variable that is conditionanlly independent from Y.  Note that X and Z are independent in this scenario.
 robustness_1_dv_jobs_p1: simulation_base.R 04_depvar_differential.R grid_sweep.py
-	${srun} grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict '{"N":[1000],"Bzx":[1], "Bxy":[0.7,0],"Bzy":[-0.7,0],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1_dv.feather"], "proxy_formula":["w_pred~y"],"z_bias":[-0.5]}' --outfile robustness_1_dv_jobs_p1
+	${srun} ./grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict '{"N":[1000],"Bzx":[1], "Bxy":[0.7,0],"Bzy":[-0.7,0],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1_dv.feather"], "proxy_formula":["w_pred~y"],"z_bias":[-0.5]}' --outfile robustness_1_dv_jobs_p1
 
 robustness_1_dv_jobs_p2: simulation_base.R 04_depvar_differential.R grid_sweep.py
-	${srun} grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict '{"N":[5000],"Bzx":[1], "Bxy":[0.7,0],"Bzy":[-0.7,0],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1_dv.feather"], "proxy_formula":["w_pred~y"],"z_bias":[-0.5]}' --outfile robustness_1_dv_jobs_p2
+	${srun} ./grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict '{"N":[5000],"Bzx":[1], "Bxy":[0.7,0],"Bzy":[-0.7,0],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1_dv.feather"], "proxy_formula":["w_pred~y"],"z_bias":[-0.5]}' --outfile robustness_1_dv_jobs_p2
 
 robustness_1_dv.feather: robustness_1_dv_jobs_p1 robustness_1_dv_jobs_p2
 	rm -f $@
@@ -136,21 +131,21 @@ robustness_1_dv.RDS: robustness_1_dv.feather summarize_estimator.R
 	${srun} Rscript plot_dv_example.R --infile $< --name "robustness_1_dv" --remember-file $@
 
 
-robustness_2_jobs_p1: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_2_jobs_p1: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.60,0.65]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.60,0.65]}' --outfile $@
 
-robustness_2_jobs_p2: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_2_jobs_p2: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.70,0.75]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.70,0.75]}' --outfile $@
 
-robustness_2_jobs_p3: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_2_jobs_p3: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.80,0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.80,0.85]}' --outfile $@
 
-robustness_2_jobs_p4: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_2_jobs_p4: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.90,0.95]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.90,0.95]}' --outfile $@
 
 robustness_2.feather: robustness_2_jobs_p1 robustness_2_jobs_p2 robustness_2_jobs_p3 robustness_2_jobs_p4
 	rm $@
@@ -172,21 +167,21 @@ robustness_2.RDS: plot_example.R robustness_2.feather summarize_estimator.R
 	rm -f $@
 	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_2" --remember-file $@
 
-robustness_2_dv_jobs_p1: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+robustness_2_dv_jobs_p1: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.60,0.65]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.60,0.65]}' --outfile $@
 
-robustness_2_dv_jobs_p2: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+robustness_2_dv_jobs_p2: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.70,0.75]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.70,0.75]}' --outfile $@
 
-robustness_2_dv_jobs_p3: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+robustness_2_dv_jobs_p3: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.80,0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.80,0.85]}' --outfile $@
 
-robustness_2_dv_jobs_p4: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+robustness_2_dv_jobs_p4: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.90,0.95]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.90,0.95]}' --outfile $@
 
 robustness_2_dv.feather: robustness_2_dv_jobs_p1 robustness_2_dv_jobs_p2 robustness_2_dv_jobs_p3 robustness_2_dv_jobs_p4
 	rm -f $@
@@ -209,9 +204,9 @@ robustness_2_dv.RDS: plot_dv_example.R robustness_2_dv.feather summarize_estimat
 	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_2_dv" --remember-file $@
 
 
-robustness_3_proflik_jobs: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_3_proflik_jobs: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":[1000],"m":[100], "seed":${seeds}, "outfile":["robustness_3_proflik.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.5,0.6,0.7,0.8,0.9,0.95], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85], "confint_method":['spline']}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":[1000],"m":[100], "seed":${seeds}, "outfile":["robustness_3_proflik.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.5,0.6,0.7,0.8,0.9,0.95], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85], "confint_method":['spline']}' --outfile $@
 
 robustness_3_proflik.feather: robustness_3_proflik_jobs
 	rm -f $@
@@ -224,17 +219,17 @@ robustness_3_proflik.RDS: plot_example.R robustness_3_proflik.feather summarize_
 	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3_proflik" --remember-file $@
 
 
-robustness_3_jobs_p1: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_3_jobs_p1: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.5,0.6], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.5,0.6], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
 
-robustness_3_jobs_p2: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_3_jobs_p2: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.7,0.8], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.7,0.8], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
 
-robustness_3_jobs_p3: grid_sweep.py 01_two_covariates.R simulation_base.R grid_sweep.py
+robustness_3_jobs_p3: grid_sweep.py 01_two_covariates.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.9,0.95], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.9,0.95], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
 
 robustness_3.feather: robustness_3_jobs_p1 robustness_3_jobs_p2 robustness_3_jobs_p3
 	rm -f $@
@@ -253,9 +248,9 @@ robustness_3.RDS: plot_example.R robustness_3.feather summarize_estimator.R
 	rm -f $@
 	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3" --remember-file $@
 
-robustness_3_dv_proflik_jobs: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+robustness_3_dv_proflik_jobs: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":[1000],"m":[100], "seed":${seeds}, "outfile":["robustness_3_dv_proflik.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0,0.405,0.846,1.386,2.197,2.944], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"confint_method":['spline']}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":[1000],"m":[100], "seed":${seeds}, "outfile":["robustness_3_dv_proflik.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0,0.405,0.846,1.386,2.197,2.944], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"confint_method":['spline']}' --outfile $@
 
 robustness_3_dv_proflik.feather: robustness_3_dv_proflik_jobs
 	rm -f $@
@@ -268,20 +263,18 @@ robustness_3_dv_proflik.RDS: plot_dv_example.R robustness_3_dv_proflik.feather s
 	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3_dv_proflik" --remember-file $@
 
 
- robustness_3_dv_jobs_p1: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+ robustness_3_dv_jobs_p1: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0,0.405], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0,0.405], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
 
-
+robustness_3_dv_jobs_p2: grid_sweep.py 03_depvar.R simulation_base.R
-
-robustness_3_dv_jobs_p2: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0.847,1.386], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0.847,1.386], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
 
 
-robustness_3_dv_jobs_p3: grid_sweep.py 03_depvar.R simulation_base.R grid_sweep.py
+robustness_3_dv_jobs_p3: grid_sweep.py 03_depvar.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "B0":[2.197,2.944], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "B0":[2.197,2.944], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
 
 robustness_3_dv.feather: robustness_3_dv_jobs_p1 robustness_3_dv_jobs_p2 robustness_3_dv_jobs_p3
 	rm -f $@
@@ -303,17 +296,22 @@ robustness_3_dv.RDS: plot_dv_example.R robustness_3_dv.feather summarize_estimat
 
 
 
-robustness_4_jobs_p1: grid_sweep.py 02_indep_differential.R simulation_base.R grid_sweep.py
+robustness_4_jobs_p1: grid_sweep.py 02_indep_differential.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 02_indep_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85],"y_bias":[-2.944,-2.197]}' --outfile $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[-2.944,-2.197]}' --outfile $@
 
-robustness_4_jobs_p2: grid_sweep.py 02_indep_differential.R simulation_base.R grid_sweep.py
+robustness_4_jobs_p2: grid_sweep.py 02_indep_differential.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 02_indep_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85], "y_bias":[-1.386,-0.846]}' --outfile $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[-1.386,-0.846]}' --outfile $@
+
+robustness_4_jobs_p3: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f ./$@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[-0.405,-0.25]}' --outfile $@
+
+robustness_4_jobs_p4: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f ./$@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[0,-0.1]}' --outfile $@
 
-robustness_4_jobs_p3: grid_sweep.py 02_indep_differential.R simulation_base.R grid_sweep.py
-	rm -f $@
-	${srun} $< --command 'Rscript 02_indep_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85],"y_bias":[-0.405,-0.25]}' --outfile $@
 
 robustness_4.feather: robustness_4_jobs_p1 robustness_4_jobs_p2 robustness_4_jobs_p3
 	rm -f $@
@@ -323,10 +321,15 @@ robustness_4.feather: robustness_4_jobs_p1 robustness_4_jobs_p2 robustness_4_job
 	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
 	$(eval END_3!=cat robustness_4_jobs_p3 | wc -l)
 	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_4_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_4))
+
 
 	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p1;)
 	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p2;)
 	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p3;)
+
 
 robustness_4.RDS: plot_example.R robustness_4.feather summarize_estimator.R
 	rm -f $@
@@ -335,34 +338,32 @@ robustness_4.RDS: plot_example.R robustness_4.feather summarize_estimator.R
 
 # '{"N":${robustness_Ns},"Bxy":[0.7],"Bzy":[-0.7],"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[0.5]}' --example_4_jobs
 
-robustness_4_dv_jobs_p1: grid_sweep.py 04_depvar_differential.R simulation_base.R grid_sweep.py
+robustness_4_dv_jobs_p1: grid_sweep.py 04_depvar_differential.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"z_bias":[0,0.1]}' --outfile $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "z_bias":[0,0.1]}' --outfile $@
 
-robustness_4_dv_jobs_p2: grid_sweep.py 04_depvar_differential.R simulation_base.R grid_sweep.py
+robustness_4_dv_jobs_p2: grid_sweep.py 04_depvar_differential.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"z_bias":[0.25,0.405]}' --outfile $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "z_bias":[0.25,0.405]}' --outfile $@
 
-robustness_4_dv_jobs_p3: grid_sweep.py 04_depvar_differential.R simulation_base.R grid_sweep.py
+robustness_4_dv_jobs_p3: grid_sweep.py 04_depvar_differential.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1],"outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"z_bias":[0.846,1.386]}' --outfile $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1],"outcome_formula":["y~x+z"],"z_bias":[0.846,1.386]}' --outfile $@ 
 
-robustness_4_dv_jobs_p4: grid_sweep.py 04_depvar_differential.R simulation_base.R grid_sweep.py
+robustness_4_dv_jobs_p4: grid_sweep.py 04_depvar_differential.R simulation_base.R
 	rm -f $@
-	${srun} $< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"],"Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"z_bias":[2.197,2.944]}' --outfile $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"],"Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "z_bias":[2.197,2.944]}' --outfile $@
 
 robustness_4_dv.feather: robustness_4_dv_jobs_p1 robustness_4_dv_jobs_p2 robustness_4_dv_jobs_p3 robustness_4_dv_jobs_p4
 	rm -f $@
 	$(eval END_1!=cat robustness_4_dv_jobs_p1 | wc -l)
 	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
-	$(eval END_2!=cat robustness_4_dv_p2 | wc -l)
+	$(eval END_2!=cat robustness_4_dv_jobs_p2 | wc -l)
 	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
-	$(eval END_3!=cat robustness_4_dv_p3 | wc -l)
+	$(eval END_3!=cat robustness_4_dv_jobs_p3 | wc -l)
 	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
-	$(eval END_3!=cat robustness_4_dv_p4 | wc -l)
+	$(eval END_4!=cat robustness_4_dv_jobs_p4 | wc -l)
-	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
-
-
 	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p1;)
 	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p2;)
 	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p3;)
@@ -371,7 +372,86 @@ robustness_4_dv.feather: robustness_4_dv_jobs_p1 robustness_4_dv_jobs_p2 robustn
 
 robustness_4_dv.RDS: plot_dv_example.R robustness_4_dv.feather summarize_estimator.R
 	rm -f $@
-	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_4" --remember-file $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_4_dv" --remember-file $@
+
+
+robustness_5_jobs_p1: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1.386,2.197], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+robustness_5_jobs_p2: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.405,0.846], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+robustness_5_jobs_p3: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0,0.25], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+robustness_5_jobs_p4: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[2.944], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+
+robustness_5.feather: robustness_5_jobs_p1 robustness_5_jobs_p2 robustness_5_jobs_p3
+	rm -f $@
+	$(eval END_1!=cat robustness_5_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_5_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_5_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_5_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p4;)
+
+robustness_5.RDS: plot_example.R robustness_5.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_5" --remember-file $@
+
+
+# '{"N":${robustness_Ns},"Bxy":[0.7],"Bzy":[-0.7],"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[0.5]}' --example_4_jobs
+
+robustness_5_dv_jobs_p1: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	 ${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[0,0.25], "outcome_formula":["y~x+z"], "z_bias":[-0.5]}' --outfile $@
+
+robustness_5_dv_jobs_p2: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[0.405,0.846], "outcome_formula":["y~x+z"],  "z_bias":[-0.5]}' --outfile $@
+
+robustness_5_dv_jobs_p3: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1.386,2.197],"outcome_formula":["y~x+z"], "z_bias":[-0.5]}' --outfile $@ 
+
+robustness_5_dv_jobs_p4: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"],"Bzy":[-0.7],"Bxy":[0.7],"Bzx":[2.944], "outcome_formula":["y~x+z"], "z_bias":[-0.5]}' --outfile $@
+
+robustness_5_dv.feather: robustness_5_dv_jobs_p1 robustness_5_dv_jobs_p2 robustness_5_dv_jobs_p3 robustness_5_dv_jobs_p4
+	rm -f $@
+	$(eval END_1!=cat robustness_5_dv_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_5_dv_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_5_dv_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_5_dv_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p4;)
+
+
+robustness_5_dv.RDS: plot_dv_example.R robustness_5_dv.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_5_dv" --remember-file $@
 
 
 clean_main:
@@ -404,5 +484,44 @@ clean_all:
 # 	sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_mecor_jobs
 # 	sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_mecor_jobs
 
+robustness_6_jobs_p1: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[0,1,2.5]}' --outfile $@
+
+robustness_6_jobs_p2: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[5,10]}' --outfile $@
+
+robustness_6_jobs_p3: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[0,1,2.5],"y_bias":[0]}' --outfile $@
+
+robustness_6_jobs_p4: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[5,10],"y_bias":[0]}' --outfile $@
+
+
+robustness_6.feather: robustness_6_jobs_p1 robustness_6_jobs_p2 robustness_6_jobs_p3 robustness_6_jobs_p4
+	rm -f $@
+	$(eval END_1!=cat robustness_6_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_6_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_6_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_6_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+
+	# $(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p1;)
+	# $(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p2;)
+	# $(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p4;)
+
+
+robustness_6.RDS: plot_example.R robustness_6.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_6" --remember-file $@
+
 
 .PHONY: supplement

simulations/measerr_methods.R

@@ -23,7 +23,7 @@ likelihood.logistic <- function(model.params, outcome, model.matrix){
 }
 
 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 
-measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){
+measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),maxit=1e6, method='optim',optim_method='L-BFGS-B'){
     df.obs <- model.frame(outcome_formula, df)
     proxy.model.matrix <- model.matrix(proxy_formula, df)
     proxy.variable <- all.vars(proxy_formula)[1]
@@ -106,7 +106,7 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
     names(start) <- params
     
     if(method=='optim'){
-        fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+        fit <- optim(start, fn = nll, lower=lower, method=optim_method, hessian=TRUE, control=list(maxit=maxit))
     } else {
         quoted.names <- gsub("[\\(\\)]",'',names(start))
         print(quoted.names)
@@ -115,13 +115,13 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
         measerr_mle_nll <- eval(parse(text=text))
         names(start) <- quoted.names
         names(lower) <- quoted.names
-        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=maxit),method=optim_method)
     }
     return(fit)
 }
 
 
-measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim', maxit=1e6, optim_method='L-BFGS-B'){
 
     df.obs <- model.frame(outcome_formula, df)
     response.var <- all.vars(outcome_formula)[1]
@@ -240,7 +240,7 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
     names(start) <- params
 
     if(method=='optim'){
-        fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+        fit <- optim(start, fn = measerr_mle_nll, lower=lower, method=optim_method, hessian=TRUE, control=list(maxit=maxit))
     } else { # method='mle2'
                 
         quoted.names <- gsub("[\\(\\)]",'',names(start))
@@ -250,7 +250,7 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
         measerr_mle_nll_mle <- eval(parse(text=text))
         names(start) <- quoted.names
         names(lower) <- quoted.names
-        fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+        fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=maxit),method=optim_method)
     }
 
     return(fit)

simulations/pl_methods.R

@@ -51,19 +51,20 @@ zhang.mle.iv <- function(df){
     fn <- df.obs[(w_pred==0) & (x.obs==1), .N]
     npv <- tn / (tn + fn)
 
+
     tp <- df.obs[(w_pred==1) & (x.obs == w_pred),.N]
     fp <- df.obs[(w_pred==1) & (x.obs == 0),.N]
     ppv <- tp / (tp + fp)
 
-    nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1){
+    nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=9){
 
     ## fpr = 1 - TNR
     ### Problem: accounting for uncertainty in ppv / npv
 
     ## fnr = 1 - TPR
     ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
-    ll <- sum(ll.y.obs)
 
+    ll <- sum(ll.y.obs)
     # unobserved case; integrate out x
     ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
     ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
@@ -75,10 +76,11 @@ zhang.mle.iv <- function(df){
     lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
 
     lls <- colLogSumExps(rbind(df.unobs$w_pred * lls.x.1, (1-df.unobs$w_pred) * lls.x.0))
+        
     ll <- ll + sum(lls)
-    return(-ll)
+
     }    
-    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf),
+    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.00001, B0=-Inf, Bxy=-Inf, Bzy=-Inf),
                    upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf),method='L-BFGS-B')
     return(mlefit)
 }

simulations/plot_dv_example.R

@@ -94,6 +94,7 @@ set.remember.prefix(gsub("plot.df.","",args$name))
 
 remember(median(sims.df$cor.xz),'med.cor.xz')
 remember(median(sims.df$accuracy),'med.accuracy')
+remember(mean(sims.df$accuracy),'mean.accuracy')
 remember(median(sims.df$error.cor.x),'med.error.cor.x')
 remember(median(sims.df$error.cor.z),'med.error.cor.z')
 remember(median(sims.df$lik.ratio),'med.lik.ratio')

simulations/run_job.sbatch

@@ -9,7 +9,6 @@
 #SBATCH --time=4:00:00
 ## Memory per node
 #SBATCH --mem=4G
-#SBATCH --cpus-per-task=1
 #SBATCH --ntasks-per-node=1
 #SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
 #SBATCH --output=simulation_jobs/%A_%a.out

simulations/simulation_base.R

@@ -151,21 +151,11 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
     temp.df <- copy(df)
     temp.df[,y:=y.obs]
 
-    if(confint_method=='quad'){
+    mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
-        mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+    fischer.info <- solve(mod.caroll.lik$hessian)
-        fischer.info <- solve(mod.caroll.lik$hessian)
+    coef <- mod.caroll.lik$par
-        coef <- mod.caroll.lik$par
+    ci.upper <- coef + sqrt(diag(fischer.info)) * 1.96
-        ci.upper <- coef + sqrt(diag(fischer.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fischer.info)) * 1.96
-        ci.lower <- coef - sqrt(diag(fischer.info)) * 1.96
-    }
-    else{ ## confint_method is 'profile'
-
-        mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, method='bbmle')
-        coef <- coef(mod.caroll.lik)
-        ci <- confint(mod.caroll.lik, method='spline')
-        ci.lower <- ci[,'2.5 %']
-        ci.upper <- ci[,'97.5 %']
-    }
 
     result <- append(result,
                      list(Bxy.est.mle = coef['x'],
@@ -175,6 +165,19 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
                           Bzy.ci.upper.mle = ci.upper['z'],
                           Bzy.ci.lower.mle = ci.lower['z']))
 
+    mod.caroll.profile.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, method='bbmle')
+    coef <- coef(mod.caroll.profile.lik)
+    ci <- confint(mod.caroll.profile.lik, method='spline')
+    ci.lower <- ci[,'2.5 %']
+    ci.upper <- ci[,'97.5 %']
+
+    result <- append(result,
+                     list(Bxy.est.mle.profile = coef['x'],
+                          Bxy.ci.upper.mle.profile = ci.upper['x'],
+                          Bxy.ci.lower.mle.profile = ci.lower['x'],
+                          Bzy.est.mle.profile = coef['z'],
+                          Bzy.ci.upper.mle.profile = ci.upper['z'],
+                          Bzy.ci.lower.mle.profile = ci.lower['z']))
 
     ## my implementatoin of liklihood based correction
     mod.zhang <- zhang.mle.dv(df)
@@ -201,8 +204,8 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
                           )
 
     tryCatch({
-        amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'))
+        amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'),ords="y.obs")
-        mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+        mod.amelia.k <- zelig(y.obs~x+z, model='logit', data=amelia.out.k$imputations, cite=FALSE)
         (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
         est.x.mi <- coefse['x','Estimate']
         est.x.se <- coefse['x','Std.Error']
@@ -340,44 +343,72 @@ run_simulation <-  function(df, result, outcome_formula=y~x+z, proxy_formula=NUL
     tryCatch({
         temp.df <- copy(df)
         temp.df <- temp.df[,x:=x.obs]
-        if(confint_method=='quad'){
+        mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula, method='optim')
-            mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula, method='optim')
+        fischer.info <- solve(mod.caroll.lik$hessian)
-            fischer.info <- solve(mod.caroll.lik$hessian)
+        coef <- mod.caroll.lik$par
-            coef <- mod.caroll.lik$par
+        ci.upper <- coef + sqrt(diag(fischer.info)) * 1.96
-            ci.upper <- coef + sqrt(diag(fischer.info)) * 1.96
+        ci.lower <- coef - sqrt(diag(fischer.info)) * 1.96
-            ci.lower <- coef - sqrt(diag(fischer.info)) * 1.96
-        } else { # confint_method == 'bbmle'
 
-            mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula, method='bbmle')
-            coef <- coef(mod.caroll.lik)
-            ci <- confint(mod.caroll.lik, method='spline')
-            ci.lower <- ci[,'2.5 %']
-            ci.upper <- ci[,'97.5 %']
-        }
         mle_result <- list(Bxy.est.mle = coef['x'],
                            Bxy.ci.upper.mle = ci.upper['x'],
                            Bxy.ci.lower.mle = ci.lower['x'],
                            Bzy.est.mle = coef['z'],
                            Bzy.ci.upper.mle = ci.upper['z'],
                            Bzy.ci.lower.mle = ci.lower['z'])
+        },
+        error=function(e) {result[['error']] <- as.character(e)
+        })
+
+    result <- append(result, mle_result)
+    mle_result_proflik <- list(Bxy.est.mle.profile = NULL,
+                               Bxy.ci.upper.mle.profile = NULL,
+                               Bxy.ci.lower.mle.profile = NULL,
+                               Bzy.est.mle.profile = NULL,
+                               Bzy.ci.upper.mle.profile = NULL,
+                               Bzy.ci.lower.mle.profile = NULL)
+
+    tryCatch({
+        ## confint_method == 'bbmle'
+        mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula, method='bbmle')
+        coef <- coef(mod.caroll.lik)
+        ci <- confint(mod.caroll.lik, method='spline')
+        ci.lower <- ci[,'2.5 %']
+        ci.upper <- ci[,'97.5 %']
+
+        mle_result_proflik <- list(Bxy.est.mle.profile = coef['x'],
+                                   Bxy.ci.upper.mle.profile = ci.upper['x'],
+                                   Bxy.ci.lower.mle.profile = ci.lower['x'],
+                                   Bzy.est.mle.profile = coef['z'],
+                                   Bzy.ci.upper.mle.profile = ci.upper['z'],
+                                   Bzy.ci.lower.mle.profile = ci.lower['z'])
     },
 
     error=function(e) {result[['error']] <- as.character(e)
     })
         
+    result <- append(result, mle_result_proflik)
 
-    result <- append(result, mle_result)
+    zhang_result <- list(Bxy.est.mle.zhang = NULL,
+                       Bxy.ci.upper.mle.zhang = NULL,
+                       Bxy.ci.lower.mle.zhang = NULL,
+                       Bzy.est.mle.zhang = NULL,
+                       Bzy.ci.upper.mle.zhang = NULL,
+                       Bzy.ci.lower.mle.zhang = NULL)
 
+    tryCatch({
     mod.zhang.lik <- zhang.mle.iv(df)
     coef <- coef(mod.zhang.lik)
     ci <- confint(mod.zhang.lik,method='quad')
-    result <- append(result,
+    zhang_result <- list(Bxy.est.zhang = coef['Bxy'],
-                     list(Bxy.est.zhang = coef['Bxy'],
+                         Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
-                          Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                         Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
-                          Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+                         Bzy.est.zhang = coef['Bzy'],
-                          Bzy.est.zhang = coef['Bzy'],
+                         Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
-                          Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+                         Bzy.ci.lower.zhang = ci['Bzy','2.5 %'])
-                          Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
+    },
+    error=function(e) {result[['error']] <- as.character(e)
+    })
+    result <- append(result, zhang_result)
 
     ## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model.
     ## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms)

simulations/summarize_estimator.R

@@ -1,5 +1,6 @@
+library(ggdist)
 
-summarize.estimator <- function(df, suffix='naive', coefname='x'){
+summarize.estimator <- function(sims.df, suffix='naive', coefname='x'){
 
     reported_vars <- c(
                        'Bxy',
@@ -13,10 +14,10 @@ summarize.estimator <- function(df, suffix='naive', coefname='x'){
 
     grouping_vars <- grouping_vars[grouping_vars %in% names(df)]
 
-    part <- df[,
+    part <- sims.df[,
-               c(reported_vars,
+                    unique(c(reported_vars,
-                 grouping_vars),
+                             grouping_vars)),
-               with=FALSE]
+                    with=FALSE]
 
 
     true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
@@ -29,6 +30,7 @@ summarize.estimator <- function(df, suffix='naive', coefname='x'){
                        bias=bias,
                        sign.correct =sign.correct)]
 
+
     part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
                           mean.bias = mean(bias),
                           mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]],na.rm=T),