changes from klone

Also fix a possible bug in the MI logic and simplify the error
correction formula in example 2.

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "paper"]
+	path = paper
+	url = git@github.com:chainsawriot/measure.git
+[submodule "overleaf"]
+	path = overleaf
+	url = https://git.overleaf.com/62a956eb9b9254783cc84c82
+[submodule "misclassificationmodels"]
+	path = misclassificationmodels
+	url = https://github.com/chainsawriot/misclassificationmodels.git

Makefile

@@ -0,0 +1,3 @@
+all:
+	+$(MAKE) -C simulations
+	+$(MAKE) -C civil_comments

charts/.gitignore

@@ -0,0 +1,9 @@
+*.aux
+*.svg
+*.pdf
+*.png
+*.aux
+*.fdb_latexmk
+*.log
+*.fls
+auto/*

charts/Makefile

@@ -6,8 +6,7 @@ all: $(patsubst %.tex,%.svg,$(wildcard *.tex)) $(patsubst %.tex,%.png,$(wildcard
 	convert -density 300 -transparent white $< $@
 
 %.svg: %.pdf
-	/usr/bin/inkscape $< --export-plain-svg=$@
+	/usr/bin/inkscape $< --export-plain-svg --export-type=svg --export-filename=$@
-
 
 %.pdf: %.tex 
 	latexmk -f -pdf $<
@@ -18,6 +17,7 @@ clean:
 	rm -f vc
 	rm *.svg
 
+
 viewpdf: all
 	evince *.pdf
 

charts/example_1_dag.tex

@@ -0,0 +1,46 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows, }
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+\begin{tikzpicture}
+
+  \node[observed] (y) {$Y$};
+  \node[unobserved, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+%  \node[unobserved, above=of w] (k) {$K$};
+  \node[observed,right=of x] (z) {$Z$};
+%   \node[residual,below=of y] (e) {$\varepsilon$};
+%  \node[residual,below=of w] (xi) {$\xi$};
+
+  \draw[->] (z) to (y);
+  \draw[->] (z) -- (x);
+  \draw[->] (x) -- (y);
+  \draw[->] (x) -- (w);
+%  \draw[->] (y) -- (w);
+%  \draw[->] (x) -- (xi);
+ % \draw[->] (w) -- (xi);
+\end{tikzpicture}
+\end{document}
+

charts/example_1_dag/.gitignore

@@ -1,12 +0,0 @@
-/auto/
-/vc
-/refs-processed.bib
-/*.aux
-/*.bbl
-/*.blg
-/*.log
-/*.out
-/*.pdf
-/*.fdb_latexmk
-/*.fls
-/*.ttt

charts/example_1_dag/Makefile

@@ -1,28 +0,0 @@
-#!/usr/bin/make
-
-all: $(patsubst %.tex,%.svg,$(wildcard *.tex)) $(patsubst %.tex,%.png,$(wildcard *.tex)) 
-
-%.png: %.pdf
-	convert -density 300 -transparent white $< $@
-
-%.svg: %.pdf
-	/usr/bin/inkscape $< --export-plain-svg=$@
-
-%.pdf: %.tex 
-	latexmk -f -pdf $<
-
-clean: 
-	latexmk -C *.tex
-	rm -f *.tmp
-	rm -f vc
-	rm *.svg
-
-viewpdf: all
-	evince *.pdf
-
-vc:
-	vc-git
-
-pdf: all
-
-.PHONY: clean all

charts/example_1_dag/example_1_dag.tex

@@ -1,46 +0,0 @@
-\documentclass[12pt]{standalone}
-
-\usepackage{ucs}
-\usepackage[utf8x]{inputenc}
-
-\usepackage[T1]{fontenc}
-\usepackage{textcomp}
-\renewcommand{\rmdefault}{ugm}
-\renewcommand{\sfdefault}{phv}
-\usepackage[garamond]{mathdesign}
-\usepackage{tikz}
-\usetikzlibrary{positioning, shapes, arrows, shadows, }
-
-\begin{document}
-
-\tikzset{>=latex}
-\begin{tikzpicture}[
-  observed/.style={circle, draw},
-  unobserved/.style={draw, circle, fill=gray!40},
-  legend/.style={rectangle, draw},
-  partly observed/.style 2 args={draw, fill=#2, path picture={
-      \fill[#1, sharp corners] (path picture bounding box.south west) -|
-      (path picture bounding box.north east) -- cycle;},
-     circle}
-  ]
-
-
-  \node[observed] (y) {Y};
-%   \node[observed,above=of y, xshift=1cm] (r) {R};
-  \node[partly observed={white}{gray!40}, above=of y] (x) {X};
-  \node[observed, left=of x,xshift=-1] (w) {W};
-  \node[unobserved, above=of w, xshift=1cm] (k) {K};
-  
-  \matrix [draw, below, yshift=-0.2cm, font=\small, align=center, column sep=2\pgflinewidth, inner sep=0.6em, outer sep=0em, nodes={align=center, anchor=center}] at (current bounding box.south){
-    \node[observed,label=right:observed] {}; \\
-    \node[unobserved,label=right:unobserved]{}; \\
-  };
-
-  \draw[->] (x) -- (y);
-  \draw[->] (x) -- (w);
-  \draw[-] (k) -- (x);
-  \draw[->] (k) -- (w);
-
-\end{tikzpicture}
-\end{document}
-

charts/example_2_dag.tex

@@ -0,0 +1,47 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+\begin{tikzpicture}
+  \node[observed] (y) {$Y$};
+  \node[unobserved, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+  \node[observed,right=of x] (z) {$Z$};
+
+  \draw[->] (z) to (y);
+  \draw[->] (z) -- (x);
+  \draw[->] (x) -- (y);
+  \draw[->] (x) -- (w);
+
+  \draw[->] (x) to (y);
+
+  \draw[->] (y) -- (w);
+  \draw[->] (y) -- (w);
+
+
+\end{tikzpicture}
+\end{document}
+

charts/example_2_dag/.gitignore

@@ -1,12 +0,0 @@
-/auto/
-/vc
-/refs-processed.bib
-/*.aux
-/*.bbl
-/*.blg
-/*.log
-/*.out
-/*.pdf
-/*.fdb_latexmk
-/*.fls
-/*.ttt

charts/example_2_dag/Makefile

@@ -1,28 +0,0 @@
-#!/usr/bin/make
-
-all: $(patsubst %.tex,%.svg,$(wildcard *.tex)) $(patsubst %.tex,%.png,$(wildcard *.tex)) 
-
-%.png: %.pdf
-	convert -density 300 -transparent white $< $@
-
-%.svg: %.pdf
-	/usr/bin/inkscape $< --export-plain-svg=$@
-
-%.pdf: %.tex 
-	latexmk -f -pdf $<
-
-clean: 
-	latexmk -C *.tex
-	rm -f *.tmp
-	rm -f vc
-	rm *.svg
-
-viewpdf: all
-	evince *.pdf
-
-vc:
-	vc-git
-
-pdf: all
-
-.PHONY: clean all

charts/example_2_dag/example_2_dag.tex

@@ -1,47 +0,0 @@
-\documentclass[12pt]{standalone}
-
-\usepackage{ucs}
-\usepackage[utf8x]{inputenc}
-
-\usepackage[T1]{fontenc}
-\usepackage{textcomp}
-\renewcommand{\rmdefault}{ugm}
-\renewcommand{\sfdefault}{phv}
-\usepackage[garamond]{mathdesign}
-\usepackage{tikz}
-\usetikzlibrary{positioning, shapes, arrows, shadows}
-
-\begin{document}
-
-\tikzset{>=latex}
-\begin{tikzpicture}[
-  observed/.style={circle, draw},
-  partly observed/.style 2 args={draw, fill=#2, path picture={
-      \fill[#1, sharp corners] (path picture bounding box.south west) -|
-      (path picture bounding box.north east) -- cycle;},
-     circle},
-    unobserved/.style={draw, circle, fill=gray!40}
-  ]
-
-  \node[observed] (y) {Y};
-  \node[observed,above=of y, xshift=1cm] (r) {R};
-  \node[partly observed={white}{gray!40}, left=of r] (x) {X};
-  \node[observed, left=of x] (w) {W};
-  \node[unobserved, above=of w, xshift=1cm] (k) {K};
-  
-  \matrix [draw, below, yshift=-0.2cm, font=\small, align=center, column sep=2\pgflinewidth, inner sep=0.6em, outer sep=0em, nodes={align=center, anchor=center}] at (current bounding box.south){
-    \node[observed,label=right:observed] {}; \\
-    \node[unobserved,label=right:unobserved]{}; \\
-  };
-
-
-  \draw[->] (r) -- (y);
-  \draw[->] (r) -- (x);
-  \draw[->] (x) -- (y);
-  \draw[->] (x) -- (w);
-  \draw[->] (k) -- (x);
-  \draw[->] (k) -- (w);
-
-\end{tikzpicture}
-\end{document}
-

charts/example_3_dag.tex

@@ -0,0 +1,42 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+
+\begin{tikzpicture}
+  \node[unobserved] (y) {$Y$};
+
+  \node[observed, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+%  \node[unobserved, above=of w] (k) {$K$};
+  \node[observed,right=of x] (z) {$Z$};
+%   \node[residual,below=of y] (e) {$\varepsilon$};
+  % \node[residual,below=of w] (xi) {$\xi$};
+  \draw[->] (z) to (y);
+  \draw[->] (x) -- (y);
+  \draw[->] (y) -- (w);
+\end{tikzpicture}
+\end{document}
+

charts/example_3_dag/.gitignore

@@ -1,12 +0,0 @@
-/auto/
-/vc
-/refs-processed.bib
-/*.aux
-/*.bbl
-/*.blg
-/*.log
-/*.out
-/*.pdf
-/*.fdb_latexmk
-/*.fls
-/*.ttt

charts/example_3_dag/example_3_dag.tex

@@ -1,51 +0,0 @@
-\documentclass[12pt]{standalone}
-
-\usepackage{ucs}
-\usepackage[utf8x]{inputenc}
-
-\usepackage[T1]{fontenc}
-\usepackage{textcomp}
-\renewcommand{\rmdefault}{ugm}
-\renewcommand{\sfdefault}{phv}
-\usepackage[garamond]{mathdesign}
-\usepackage{tikz}
-\usetikzlibrary{positioning, shapes, arrows, shadows}
-
-\begin{document}
-
-\tikzset{>=latex}
-\begin{tikzpicture}[
-  observed/.style={circle, draw},
-  partly observed/.style 2 args={draw, fill=#2, path picture={
-      \fill[#1, sharp corners] (path picture bounding box.south west) -|
-      (path picture bounding box.north east) -- cycle;},
-     circle},
-    unobserved/.style={draw, circle, fill=gray!40}
-  ]
-
-  \node[observed] (y) {Y};
-%  \node[observed,above=of y, xshift=1cm] (r) {R};
-  \node[partly observed={white}{gray!40}, above = of y] (x) {X};
-  \node[observed, left=of x] (w) {W};
-  \node[unobserved, above=of w, xshift=1cm] (k) {K};
-  \node[unobserved, right=of k] (u) {U};
-
-%  \draw[->] (r) -- (y);
-%  \draw[->] (r) -- (x);
-  \draw[->] (x) -- (y);
-  \draw[->] (x) -- (w);
-  \draw[->] (x) -- (k);
-  \draw[->] (k) -- (w);
-  \draw[->] (u) to [out=270,in=30] (y);
-  \draw[->] (u) -- (k);
-
-  
-  \matrix [draw, below, yshift=-0.2cm, inner sep=0.6em, outer sep=0em, nodes={align=center, anchor=center}] at (current bounding box.south){
-    \node[observed,label=right:observed] {}; \\
-    \node[unobserved,label=right:unobserved]{}; \\
-  };
-
-
-\end{tikzpicture}
-\end{document}
-

charts/example_4_dag.tex

@@ -0,0 +1,47 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+
+\begin{tikzpicture}
+  \node[unobserved] (y) {$Y$};
+
+  \node[observed={white}{gray!40}, above=of y] (x) {$X$};
+  \node[observed, left=of x] (w) {$W$};
+
+%  \node[unobserved, above=of w] (k) {$K$};
+  \node[observed,right=of x] (z) {$Z$};
+%   \node[residual,below=of y] (e) {$\varepsilon$};
+%   \node[residual,below=of w] (xi) {$\xi$};
+  \draw[->] (x) -- (y);
+%  \draw[->] (x) -- (w);
+  \draw[->] (y) -- (w);
+%  \draw[->] (k) -- (w);
+  \draw[->] (z) -- (y);
+%  \draw[->] (z) -- (k);
+%  \draw[->] (y) -- (xi);
+%  \draw[->] (w) -- (xi);
+\end{tikzpicture}
+\end{document}
+

charts/example_legend.tex

@@ -0,0 +1,35 @@
+\documentclass[12pt]{standalone}
+
+\usepackage{ucs}
+\usepackage[utf8x]{inputenc}
+
+\usepackage[T1]{fontenc}
+\usepackage{textcomp}
+\renewcommand{\rmdefault}{ugm}
+\renewcommand{\sfdefault}{phv}
+\usepackage[garamond]{mathdesign}
+\usepackage{tikz}
+\usetikzlibrary{positioning, shapes, arrows, shadows}
+
+\begin{document}
+\tikzset{
+  observed/.style={circle, draw},
+  partly observed/.style 2 args={draw, fill=#2, path picture={
+      \fill[#1, sharp corners] (path picture bounding box.south west) -|
+      (path picture bounding box.north east) -- cycle;},
+     circle},
+    unobserved/.style={draw, circle, fill=gray!40},
+    residual/.style={draw, rectangle}
+}
+
+\tikzset{>=latex}
+
+\begin{tikzpicture}
+  \matrix [draw, below, font=\small, align=center, column sep=2\pgflinewidth, inner sep=0.4em, outer sep=0em, nodes={align=center, anchor=center}] at (current bounding box.south){
+    \node[observed,label=right:observed] {}; \\
+    \node[unobserved,label=right:automatically classified]{}; \\
+    \node[residual,label=right:error term]{}; \\
+  };
+\end{tikzpicture}
+\end{document}
+

civil_comments/01_dv_example.R

@@ -0,0 +1,89 @@
+source('load_perspective_data.R')
+source("../simulations/measerr_methods.R")
+source("../simulations/RemembR/R/RemembeR.R")
+
+change.remember.file("dv_perspective_example.RDS")
+remember(accuracies, "civil_comments_accuracies")
+remember(f1s, "civil_comments_f1s")
+remember(positive_cases, "civil_comments_positive_cases")
+remember(proportions_cases, "civil_comments_proportions_cases")
+remember(cortab, "civil_comments_cortab")
+remember(nrow(df), 'n.annotated.comments')
+# for reproducibility
+set.seed(111)
+
+## another simple enough example: is P(toxic | funny and white) > P(toxic | funny nand white)? Or, are funny comments more toxic when people disclose that they are white?
+
+compare_dv_models <-function(pred_formula, outcome_formula, proxy_formula, df, sample.prop, sample.size, remember_prefix){
+    if(is.null(sample.prop)){
+        sample.prop <- sample.size / nrow(df)
+    }
+    if(is.null(sample.size)){
+        sample.size <- nrow(df) * sample.prop
+    }
+
+    pred_model <- glm(pred_formula, df, family=binomial(link='logit'))
+
+    remember(sample.size, paste0(remember_prefix, "sample.size"))
+    remember(sample.prop, paste0(remember_prefix, "sample.prop"))
+    remember(pred_formula, paste0(remember_prefix, "pred_formula"))
+    remember(outcome_formula, paste0(remember_prefix, 'outcome_formula'))
+    remember(proxy_formula, paste0(remember_prefix, 'proxy_formula'))
+
+    remember(coef(pred_model), paste0(remember_prefix, "coef_pred_model"))
+    remember(diag(vcov((pred_model))), paste0(remember_prefix, "se_pred_model"))
+
+    coder_model <- glm(outcome_formula, df, family=binomial(link='logit'))
+    remember(coef(coder_model), paste0(remember_prefix, "coef_coder_model"))
+    remember(diag(vcov((coder_model))), paste0(remember_prefix, "se_coder_model"))
+
+    df_measerr_method <- copy(df)[sample(1:.N, sample.size), toxicity_coded_1 := toxicity_coded]
+    df_measerr_method <- df_measerr_method[,toxicity_coded := toxicity_coded_1]
+    sample_model <- glm(outcome_formula, df_measerr_method, family=binomial(link='logit'))
+    remember(coef(sample_model), paste0(remember_prefix, "coef_sample_model"))
+    remember(diag(vcov((sample_model))), paste0(remember_prefix, "se_sample_model"))
+
+    measerr_model <- measerr_mle_dv(df_measerr_method, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula=proxy_formula, proxy_family=binomial(link='logit'))
+
+    inv_hessian = solve(measerr_model$hessian)
+    stderr = diag(inv_hessian)
+    remember(stderr, paste0(remember_prefix, "measerr_model_stderr"))
+    remember(measerr_model$par, paste0(remember_prefix, "measerr_model_par"))
+}
+
+print("running first example")
+
+pred_formula = toxicity_pred ~ likes + race_disclosed
+outcome_formula = toxicity_coded ~ likes + race_disclosed
+proxy_formula = toxicity_pred ~ toxicity_coded*race_disclosed*likes
+
+compare_dv_models(pred_formula = pred_formula,
+                  outcome_formula = outcome_formula,
+                  proxy_formula = proxy_formula,
+                  df=df,
+                  sample.prop=0.01,
+                  sample.size=NULL,
+                  remember_prefix='cc_ex_tox.likes.race_disclosed')
+
+
+print("running second example")
+
+compare_dv_models(pred_formula = pred_formula,
+                  outcome_formula = outcome_formula,
+                  proxy_formula = proxy_formula,
+                  df=df,
+                  sample.size=10000,
+                  sample.prop=NULL,
+                  remember_prefix='cc_ex_tox.likes.race_disclosed.medsamp')
+
+
+print("running third example")
+
+compare_dv_models(pred_formula = pred_formula,
+                  outcome_formula = outcome_formula,
+                  proxy_formula = proxy_formula,
+                  df=df,
+                  sample.prop=0.05,
+                  sample.size=NULL,
+                  remember_prefix='cc_ex_tox.likes.race_disclosed.largesamp')
+

civil_comments/02_iv_example.R

@@ -0,0 +1,107 @@
+source("../simulations/RemembR/R/RemembeR.R")
+change.remember.file("iv_perspective_example.RDS")
+
+source('load_perspective_data.R')
+source("../simulations/measerr_methods.R")
+
+remember(accuracies, "civil_comments_accuracies")
+remember(f1s, "civil_comments_f1s")
+remember(positive_cases, "civil_comments_positive_cases")
+remember(proportions_cases, "civil_comments_proportions_cases")
+remember(cortab, "civil_comments_cortab")
+remember(nrow(df), 'n.annotated.comments')
+# for reproducibility
+set.seed(1)
+
+## another simple enough example: is P(toxic | funny and white) > P(toxic | funny nand white)? Or, are funny comments more toxic when people disclose that they are white?
+
+compare_iv_models <-function(pred_formula, outcome_formula, proxy_formula, truth_formula, df, sample.prop, sample.size, remember_prefix){
+
+    if(is.null(sample.prop)){
+        sample.prop <- sample.size / nrow(df)
+    }
+    if(is.null(sample.size)){
+        sample.size <- nrow(df) * sample.prop
+    }
+
+    remember(sample.size, paste0(remember_prefix, "sample.size"))
+    remember(sample.prop, paste0(remember_prefix, "sample.prop"))
+    remember(pred_formula, paste0(remember_prefix, "pred_formula"))
+    remember(outcome_formula, paste0(remember_prefix, 'outcome_formula'))
+    remember(proxy_formula, paste0(remember_prefix, 'proxy_formula'))
+    remember(truth_formula, paste0(remember_prefix, 'truth_formula'))
+
+    pred_model <- glm(pred_formula, df, family=binomial(link='logit'))
+    remember(coef(pred_model), paste0(remember_prefix, "coef_pred_model"))
+    remember(diag(vcov((pred_model))), paste0(remember_prefix, "se_pred_model"))
+
+    coder_model <- glm(outcome_formula, df, family=binomial(link='logit'))
+    remember(coef(coder_model), paste0(remember_prefix, "coef_coder_model"))
+    remember(diag(vcov((coder_model))), paste0(remember_prefix, "se_coder_model"))
+
+    df_measerr_method <- copy(df)[sample(1:.N, sample.size), toxicity_coded_1 := toxicity_coded]
+    df_measerr_method <- df_measerr_method[,toxicity_coded := toxicity_coded_1]
+    sample_model <- glm(outcome_formula, df_measerr_method, family=binomial(link='logit'))
+    remember(coef(sample_model), paste0(remember_prefix, "coef_sample_model"))
+    remember(diag(vcov((sample_model))), paste0(remember_prefix, "se_sample_model"))
+
+    measerr_model <- measerr_mle(df_measerr_method, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula=proxy_formula, proxy_family=binomial(link='logit'),truth_formula=truth_formula, truth_family=binomial(link='logit'))
+
+    inv_hessian = solve(measerr_model$hessian)
+    stderr = diag(inv_hessian)
+    remember(stderr, paste0(remember_prefix, "measerr_model_stderr"))
+    remember(measerr_model$par, paste0(remember_prefix, "measerr_model_par"))
+}
+
+## print("running first iv example")
+
+## sample.prop <- 0.05
+
+## compare_iv_models(white ~ toxicity_pred*funny,
+##                   outcome_formula = white ~ toxicity_coded*funny,
+##                   proxy_formula = toxicity_pred ~ toxicity_coded*funny*white,
+##                   truth_formula = toxicity_coded ~ 1,
+##                   df=df,
+##                   sample.prop=sample.prop,
+##                   remember_prefix='cc_ex_tox.funny.white')
+
+
+
+pred_formula <- race_disclosed ~ likes * toxicity_pred
+outcome_formula <- race_disclosed ~ likes * toxicity_coded
+proxy_formula <- toxicity_pred ~ toxicity_coded * race_disclosed * likes
+truth_formula <- toxicity_coded ~ 1
+
+print("running first example")
+
+compare_iv_models(pred_formula = pred_formula,
+                  outcome_formula = outcome_formula,
+                  proxy_formula = proxy_formula,
+                  truth_formula = truth_formula,
+                  df=df,
+                  sample.prop=0.01,
+                  sample.size=NULL,
+                  remember_prefix='cc_ex_tox.likes.race_disclosed')
+
+print("running second example")
+
+compare_iv_models(pred_formula = pred_formula,
+                  outcome_formula = outcome_formula,
+                  proxy_formula = proxy_formula,
+                  truth_formula = truth_formula,
+                  df=df,
+                  sample.prop=NULL,
+                  sample.size=10000,
+                  remember_prefix='cc_ex_tox.likes.race_disclosed.medsamp')
+
+print("running third example")
+
+compare_iv_models(pred_formula = race_disclosed ~ likes * toxicity_pred,
+                  outcome_formula = race_disclosed ~ likes * toxicity_coded,
+                  proxy_formula = toxicity_pred ~ toxicity_coded + race_disclosed,
+                  truth_formula = toxicity_coded ~ 1,
+                  df=df,
+                  sample.prop=0.05,
+                  sample.size=NULL,
+                  remember_prefix='cc_ex_tox.likes.race_disclosed.largesamp')
+

civil_comments/03_prob_not_pred.R

@@ -0,0 +1,21 @@
+source('load_perspective_data.R')
+source("../simulations/RemembR/R/RemembeR.R")
+library(xtable)
+change.remember.file("prob_not_pred.RDS")
+### to respond to the reviewer show what happens if we don't recode the predictions.
+
+non_recoded_dv <- lm(toxicity_prob ~ likes * race_disclosed, data=df)
+remember(coef(non_recoded_dv), "coef_dv")
+remember(diag(vcov(non_recoded_dv)), "se_dv")
+remember(xtable(non_recoded_dv),'dv_xtable')
+
+non_recoded_iv <- glm(race_disclosed ~ likes * toxicity_prob, data=df, family='binomial')
+remember(coef(non_recoded_iv), "coef_iv")
+remember(diag(vcov(non_recoded_iv)), "se_iv")
+remember(xtable(non_recoded_iv),'iv_xtable')
+
+remember(extract(non_recoded_iv,include.aic=F,include.bic=F,include.nobs=F,include.deviance=F,include.loglik=F),'non_recoded_iv')
+remember(extract(non_recoded_dv,include.rsquared=F,include.adjrs=F,include.nobs=F),'non_recoded_dv')
+tr <- texreg(list(r$non_recoded_iv, r$non_recoded_dv),custom.model.names=c("Example 1","Example 2"),custom.coef.map=list("(Intercept)"="Intercept","race_disclosedTRUE"="Identity Disclosure","toxicity_prob"="Toxicity Score","likes"="Likes","likes:race_disclosedTRUE"="Likes:Identity Disclosure","likes:toxicity_prob"="Likes:Toxicity Score"),single.row=T,dcolumn=T)
+print(tr)
+remember(tr, 'texregobj')

civil_comments/Makefile

@@ -0,0 +1,14 @@
+qall: iv_perspective_example.RDS dv_perspective_example.RDS
+
+srun_1core=srun -A comdata -p compute-bigmem --mem=4G --time=12:00:00 -c 1 --pty /usr/bin/bash -l
+srun=srun -A comdata -p compute-bigmem --mem=4G --time=12:00:00 --pty /usr/bin/bash -l
+
+perspective_scores.csv: perspective_json_to_csv.sh perspective_results.json
+	$(srun_1core) ./$^ $@
+
+iv_perspective_example.RDS: 02_iv_example.R perspective_scores.csv
+	$(srun) Rscript $<
+
+dv_perspective_example.RDS: 01_dv_example.R perspective_scores.csv
+	 $(srun) Rscript $<
+

civil_comments/all_data.csv

@@ -0,0 +1 @@
+../.git/annex/objects/6v/fJ/SHA256E-s916052376--a85b5ba7e9a8cda38b91ea6e3957a4f2bfff17bb52f22c935595cbe47cc54d94.csv/SHA256E-s916052376--a85b5ba7e9a8cda38b91ea6e3957a4f2bfff17bb52f22c935595cbe47cc54d94.csv

civil_comments/design_example.R

@@ -0,0 +1,72 @@
+set.seed(1111)
+source('load_perspective_data.R')
+## how accurate are the classifiers?
+
+## the API claims that these scores are "probabilities"
+## say we care about the model of the classification, not the probability
+
+## toxicity error is weakly correlated pearson's R = 0.1 with both "white" and "black".
+
+## compare regressions with "white" or "black" as the outcome and "toxicity_coded" or "toxicity_pred" as a predictor.
+## here's a great example with presumambly non-differential error: about what identities is toxicity found humorous?
+## a bunch of stars reappear when you used the ground-truth data instead of the predictions.
+## pro/con of this example: will have to implement family='poisson'.
+## shouldn't be that bad though haha.
+cortab['toxicity_error',]
+cortab['toxicity_error','funny']
+cortab['toxicity_coded',]
+cortab['identity_error',]
+cortab['white',]
+
+## here's a simple example, is P(white | toxic and mentally ill) > P(white | toxic or mentally ill). Are people who discuss their mental illness in a toxic way more likely to be white compared to those who just talk about their mental illness or are toxic? 
+summary(glm(white ~ toxicity_coded*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
+
+summary(glm(white ~ toxicity_pred*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
+
+summary(glm(white ~ toxicity_coded*male, data = df, family=binomial(link='logit')))
+
+summary(glm(white ~ toxicity_pred*male, data = df, family=binomial(link='logit')))
+
+summary(glm(toxicity_coded ~ white*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
+
+summary(glm(toxicity_pred ~ white*psychiatric_or_mental_illness, data = df, family=binomial(link='logit')))
+
+
+## another simple enough example: is P(toxic | funny and white) > P(toxic | funny nand white)? Or, are funny comments more toxic when people disclose that they are white?
+
+summary(glm(toxicity_pred ~ funny*white, data=df, family=binomial(link='logit')))
+summary(glm(toxicity_coded ~ funny*white, data=df, family=binomial(link='logit')))
+
+source("../simulations/measerr_methods.R")
+                                                                                       
+saved_model_file <- "measerr_model_tox.eq.funny.cross.white.RDS"
+overwrite_model <- TRUE 
+
+# it works so far with a 20% and 15% sample. Smaller is better. let's try a 10% sample again. It didn't work out. We'll go forward with a 15% sample.
+df_measerr_method <- copy(df)[sample(1:.N, 0.05 * .N), toxicity_coded_1 := toxicity_coded]
+df_measerr_method <- df_measerr_method[,toxicity_coded := toxicity_coded_1]
+summary(glm(toxicity_coded ~ funny*white, data=df_measerr_method[!is.na(toxicity_coded)], family=binomial(link='logit')))
+
+if(!file.exists(saved_model_file) || (overwrite_model == TRUE)){
+    measerr_model <- measerr_mle_dv(df_measerr_method,toxicity_coded ~ funny*white,outcome_family=binomial(link='logit'), proxy_formula=toxicity_pred ~ toxicity_coded*funny*white)
+saveRDS(measerr_model, saved_model_file)
+} else {
+    measerr_model <- readRDS(saved_model_file)
+}
+
+inv_hessian <- solve(measerr_model$hessian)
+se <- diag(inv_hessian)
+
+lm2 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity_pred, data = df)
+m3 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity, data = df)
+
+
+glm(white ~ disagree, data = df, family=binomial(link='logit'))
+
+## example with differential error
+
+glm(white ~ toxicity_coded + toxicity_error, data=df,family=binomial(link='logit'))
+
+glm(toxicity_coded ~ white, data = df, family=binomial(link='logit'))
+
+glm(toxicity_pred ~ white, data = df, family=binomial(link='logit'))

civil_comments/get_perspective_scores.py

@@ -0,0 +1,39 @@
+from googleapiclient import discovery
+import json
+import csv
+from pathlib import Path
+
+from time import sleep
+
+from itertools import islice
+
+API_KEY = open('perspective_api_key').read()
+
+client = discovery.build("commentanalyzer","v1alpha",developerKey=API_KEY,discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",static_discovery=False,)
+
+csvreader = csv.DictReader(open("all_data.csv"), dialect='unix')
+
+outfile = Path("perspective_results.json")
+already_scored = set()
+if outfile.exists():
+    already_scored = set([json.loads(l)['id'] for l in open(str(outfile),'r')])
+
+print(f"loaded {len(already_scored)} scored comments")
+with open("perspective_results.json",'a') as of:
+    for line in csvreader:
+        if line['id'] not in already_scored and len(line.get('comment_text','')) > 0:
+            analyze_request = {'comment':{'text':line['comment_text']},
+                               'languages':['en'],
+                               'requestedAttributes':{'TOXICITY':{},
+                                                      "SEVERE_TOXICITY":{},
+                                                      "IDENTITY_ATTACK":{},
+                                                      "INSULT":{},
+                                                      "PROFANITY":{},
+                                                      "THREAT":{}}}
+            response = client.comments().analyze(body=analyze_request).execute()
+            response['id'] = line['id']
+            result = json.dumps(response)
+            of.write(result + '\n')
+            of.flush()
+
+            sleep(0.10)

civil_comments/identity_individual_annotations.csv

@@ -0,0 +1 @@
+../.git/annex/objects/qP/Xw/SHA256E-s106388260--7b8e9f21c5110d32e337137f8b4fe50987ec1b59fdbfd56a4717cdc13e509ec3.csv/SHA256E-s106388260--7b8e9f21c5110d32e337137f8b4fe50987ec1b59fdbfd56a4717cdc13e509ec3.csv

civil_comments/load_perspective_data.R

@@ -0,0 +1,136 @@
+library(data.table)
+library(MASS)
+
+set.seed(1111)
+
+scores <- fread("perspective_scores.csv")
+scores <- scores[,id:=as.character(id)]
+
+df <- fread("all_data.csv")
+
+# only use the data that has identity annotations
+df <- df[identity_annotator_count > 0]
+
+(df[!(df$id %in% scores$id)])
+
+df <- df[scores,on='id',nomatch=NULL]
+
+df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
+          insult_pred = insult_prob >= 0.5,
+          profanity_pred = profanity_prob >= 0.5,
+          severe_toxicity_pred = severe_toxicity_prob >= 0.5,
+          threat_pred = threat_prob >= 0.5,
+          toxicity_pred = toxicity_prob >= 0.5,
+          identity_attack_coded = identity_attack >= 0.5,
+          insult_coded = insult >= 0.5,
+          profanity_coded = obscene >= 0.5,
+          severe_toxicity_coded = severe_toxicity >= 0.5,
+          threat_coded = threat >= 0.5,
+          toxicity_coded = toxicity >= 0.5
+          )]
+
+gt.0.5 <- function(v) { v >= 0.5 }
+dt.apply.any <- function(fun, ...){apply(apply(cbind(...), 2, fun),1,any)}
+
+df <- df[,":="(gender_disclosed = dt.apply.any(gt.0.5, male, female, transgender, other_gender),
+               sexuality_disclosed = dt.apply.any(gt.0.5, heterosexual, bisexual, other_sexual_orientation),
+               religion_disclosed = dt.apply.any(gt.0.5, christian, jewish, hindu, buddhist, atheist, muslim, other_religion),
+               race_disclosed = dt.apply.any(gt.0.5, white, black, asian, latino, other_race_or_ethnicity), 
+               disability_disclosed = dt.apply.any(gt.0.5,physical_disability, intellectual_or_learning_disability, psychiatric_or_mental_illness, other_disability))]
+
+df <- df[,white:=gt.0.5(white)]
+
+
+F1 <- function(y, predictions){
+    tp <- sum( (predictions == y) & (predictions==1))
+    fn <- sum( (predictions != y) & (predictions!=1))
+    fp <- sum( (predictions != y) & (predictions==1))
+    precision <- tp / (tp + fp)
+    recall <- tp / (tp + fn)
+    return (2 * precision * recall ) / (precision + recall)
+}
+
+
+## toxicity is about 93% accurate, with an f1 of 0.8
+## identity_attack has high accuracy 97%, but an unfortunant f1 of 0.5.
+## threat has high accuracy 99%, but a really bad looking f1 of 0.48.
+accuracies <- df[,.(identity_attack_acc = mean(identity_attack_pred == identity_attack_coded),
+                    insult_pred_acc = mean(insult_pred == insult_coded),
+                    profanity_acc = mean(profanity_pred == profanity_coded),
+                    severe_toxicity_acc = mean(severe_toxicity_pred == severe_toxicity_coded),
+                    theat_acc = mean(threat_pred == threat_coded),
+                    toxicity_acc = mean(toxicity_pred == toxicity_coded))]
+
+f1s <- df[,.(identity_attack_f1 = F1(identity_attack_coded,identity_attack_pred),
+                    insult_f1 = F1(insult_coded,insult_pred),
+                    profanity_f1 = F1(profanity_coded,profanity_pred),
+                    severe_toxicity_f1 = F1(severe_toxicity_coded,severe_toxicity_pred),
+                    theat_f1 = F1(threat_coded,threat_pred),
+                    toxicity_f1 = F1(toxicity_coded,toxicity_pred))]
+
+positive_cases <- df[,.(identity_attacks = sum(identity_attack_coded),
+                        insults = sum(insult_coded),
+                        profanities = sum(profanity_coded),
+                        severe_toxic_comments = sum(severe_toxicity_coded),
+                        threats = sum(threat_coded),
+                        toxic_comments = sum(toxicity_coded))]
+
+## there are 50,000 toxic comments, 13000 identity attacks, 30000 insults, 3000 profanities, 8 severe toxic, and 1000 threats.
+
+proportions_cases <- df[,.(prop_identity = mean(identity_attack_coded),
+                           prop_insults = mean(insult_coded),
+                           prop_profanity = mean(profanity_coded),
+                           prop_severe = mean(severe_toxicity_coded),
+                           prop_threats = mean(threat_coded),
+                           prop_toxic = mean(toxicity_coded))]
+
+## at 11% of comments, "toxicity" seems not so badly skewed. Try toxicity first, and if it doesn't work out try insults.
+
+## now look for an example where differential error affects an identity, or a reaction.
+df <- df[,":="(identity_error = identity_attack_coded - identity_attack_pred,
+               insult_error = insult_coded - insult_pred,
+               profanity_error = profanity_coded - profanity_pred,
+               severe_toxic_error = severe_toxicity_coded - severe_toxicity_pred,
+               threat_error = threat_coded - threat_pred,
+               toxicity_error = toxicity_coded - toxicity_pred)]
+
+## what's correlated with toxicity_error ?
+df <- df[,approved := rating == "approved"]
+df <- df[,white := white > 0.5]
+
+cortab <- cor(df[,.(toxicity_error,
+                    identity_error,
+                    toxicity_coded,
+                    funny,
+                    approved,
+                    sad,
+                    wow,
+                    likes,
+                    disagree,
+                    male,
+                    female,
+                    transgender,
+                    other_gender,
+                    heterosexual,
+                    bisexual,
+                    other_sexual_orientation,
+                    christian,
+                    jewish,
+                    hindu,
+                    buddhist,
+                    atheist,
+                    other_religion,
+                    black,
+                    white,
+                    asian,
+                    latino,
+                    other_race_or_ethnicity,
+                    physical_disability,
+                    intellectual_or_learning_disability,
+                    psychiatric_or_mental_illness,
+                    other_disability,
+                    gender_disclosed,
+                    sexuality_disclosed,
+                    religion_disclosed,
+                    race_disclosed,
+                    disability_disclosed)])

civil_comments/perspective_json_to_csv.jq

@@ -0,0 +1,2 @@
+#!/usr/bin/bash
+cat $1 | jq "[.attributeScores.IDENTITY_ATTACK.summaryScore.value, .attributeScores.INSULT.summaryScore.value, .attributeScores.PROFANITY.summaryScore.value,.attributeScores.SEVERE_TOXICITY.summaryScore.value, .attributeScores.THREAT.summaryScore.value, .attributeScores.TOXICITY.summaryScore.value] | @csv" > $2

civil_comments/perspective_json_to_csv.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/bash
+header=id,identity_attack_prob,insult_prob,profanity_prob,severe_toxicity_prob,threat_prob,toxicity_prob
+echo "$header" > $2
+cat $1 | jq -r '[.id, .attributeScores.IDENTITY_ATTACK.summaryScore.value, .attributeScores.INSULT.summaryScore.value, .attributeScores.PROFANITY.summaryScore.value,.attributeScores.SEVERE_TOXICITY.summaryScore.value, .attributeScores.THREAT.summaryScore.value, .attributeScores.TOXICITY.summaryScore.value] | @csv' >>  $2

civil_comments/perspective_results.json

@@ -0,0 +1 @@
+../.git/annex/objects/ZV/z8/SHA256E-s2293825121--6cdc8f8fb64fad2e51027e2564928e8938bf5fc6ca0cd6c31cb2e67aafe0a203.json/SHA256E-s2293825121--6cdc8f8fb64fad2e51027e2564928e8938bf5fc6ca0cd6c31cb2e67aafe0a203.json

civil_comments/toxicity_individual_annotations.csv

@@ -0,0 +1 @@
+../.git/annex/objects/FF/WZ/SHA256E-s417648663--c85bda15b964a24869ae11f76092bde6f4b18236dd1cbe17539526b3b5b736cf.csv/SHA256E-s417648663--c85bda15b964a24869ae11f76092bde6f4b18236dd1cbe17539526b3b5b736cf.csv

simulations/.Rhistory

@@ -63,3 +63,94 @@ list.files()
 install.packages("filelock")
 q()
 n
+df
+df
+outcome_formula <- y ~ x + z
+outcome_family=gaussian()
+proxy_formula <- w_pred ~ x
+truth_formula <- x ~ z
+params <- start
+ll.y.obs.x0
+ll.y.obs.x1
+rater_formula <- x.obs ~ x
+rater_formula
+rater.modle.matrix.obs.x0
+rater.model.matrix.obs.x0
+names(rater.model.matrix.obs.x0)
+head(rater.model.matrix.obs.x0)
+df.obs
+ll.x.obs.0
+rater.params
+rater.params %*% t(rater.model.matrix.x.obs.0[df.obs$xobs.0==1])
+df.obs$xobs.0==1
+df.obs$x.obs.0==1
+ll.x.obs.0[df.obs$x.obs.0==1]
+rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]
+df.obs$x.obs.0==1
+n.rater.model.covars <- dim(rater.model.matrix.x.obs.0)[2]
+        rater.params <- params[param.idx:n.rater.model.covars]
+rater.params
+        ll.x.obs.0[df.obs$x.obs.0==1] <- plogis(rater.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]), log=TRUE)
+t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]
+)
+dimt(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,])
+dim(t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]))
+dim(ll.x.obs.0[df.obs$x.obs.0==1])
+rater.params
+rater.params
+rater.params
+rater_formula
+rater.params
+)
+1+1
+q()
+n
+outcome_formula <- y ~ x + z
+proxy_formula <- w_pred ~ x + z + y
+truth_formula <- x ~ z
+proxy_formula
+eyboardio Model 01 - Kaleidoscope locally built
+df <- df.triple.proxy.mle
+outcome_family='gaussian'
+outcome_family=gaussian()
+proxy_formulas=list(proxy_formula,x.obs.0~x, x.obs.1~x)
+proxy_formulas
+proxy_familites <- rep(binomial(link='logit'),3)
+proxy_families = rep(binomial(link='logit'),3)
+proxy_families
+proxy_families = list(binomial(link='logit'),binomial(link='logit'),binomial(link='logit'))
+proxy_families
+proxy_families[[1]]
+proxy.params
+i
+proxy_params
+proxy.params
+params
+params <- start
+df.triple.proxy.mle
+df
+coder.formulas <- c(x.obs.0 ~ x, x.obs.1 ~x)
+outcome.formula
+outcome_formula
+depvar(outcome_formula
+)
+outcome_formula$terms
+terms(outcome_formula)
+q()
+n
+df.triple.proxy.mle
+triple.proxy.mle
+df
+df <- df.triple.proxy
+outcome_family <- binomial(link='logit')
+outcome_formula <- y ~x+z
+proxy_formula <- w_pred ~ y
+coder_formulas=list(y.obs.1~y,y.obs.2~y); proxy_formula=w_pred~y; proxy_family=binomial(link='logit'))
+coder_formulas=list(y.obs.1~y,y.obs.2~y); proxy_formula=w_pred~y; proxy_family=binomial(link='logit')
+coder_formulas=list(y.obs.0~y,y.obs.1~y)
+traceback()
+df
+df
+outcome.model.matrix
+q()
+n

simulations/01_two_covariates.R

@@ -30,11 +30,11 @@ source("simulation_base.R")
 #### how much power do we get from the model in the first place? (sweeping N and m)
 #### 
 
-simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, seed=1){
+simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, Px=0.5, seed=1){
     set.seed(seed)
-    z <- rbinom(N, 1, 0.5)
+    z <- rnorm(N,sd=0.5)
                                         #    x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance)
-    xprime <- Bzx * z #+ x.var.epsilon
+    xprime <- Bzx * z + qlogis(Px)
     x <- rbinom(N,1,plogis(xprime))
 
     y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance)
@@ -71,31 +71,38 @@ parser <- add_argument(parser, "--outfile", help='output file', default='example
 parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05)
 # parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)
 parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
-parser <- add_argument(parser, "--Bzx", help='coefficient of z on x?', default=1)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
-args <- parse_args(parser)
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~x")
 
+parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
+parser <- add_argument(parser, "--Bxy", help='Effect of x on y', default=0.3)
+parser <- add_argument(parser, "--Px", help='Base rate of x', default=0.5)
+parser <- add_argument(parser, "--confint_method", help='method for approximating confidence intervals', default='quad')
+
+args <- parse_args(parser)
 B0 <- 0
-Bxy <- 0.3
+Px <- args$Px
-Bzy <- -0.3
+Bxy <- args$Bxy
+Bzy <- args$Bzy
 Bzx <- args$Bzx
 
-if (args$m < args$N){
+df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, Px, seed=args$seed + 500, y_explained_variance = args$y_explained_variance,  prediction_accuracy=args$prediction_accuracy)
 
-    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance,  prediction_accuracy=args$prediction_accuracy)
+result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, 'Bzx'=Bzx, 'Bzy'=Bzy, 'Px'=Px, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'outcome_formula'=args$outcome_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, confint_method=args$confint_method,error='')
 
-    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'zx_explained_variance' = args$zx_explained_variance, "prediction_accuracy"=args$prediction_accuracy, "error"="")
+outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula),confint_method=args$confint_method)
     
-    outline <- run_simulation(df, result)
+outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
-    
+if(file.exists(args$outfile)){
-    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    logdata <- read_feather(args$outfile)
-    if(file.exists(args$outfile)){
+    logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
-        logdata <- read_feather(args$outfile)
+} else {
-        logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+    logdata <- as.data.table(outline)
-    } else {
-        logdata <- as.data.table(outline)
-    }
-
-    print(outline)
-    write_feather(logdata, args$outfile)
-    unlock(outfile_lock)
 }
+
+print(outline)
+write_feather(logdata, args$outfile)
+unlock(outfile_lock)
+

simulations/02_indep_differential.R

@@ -31,11 +31,11 @@ source("simulation_base.R")
 
 ## one way to do it is by adding correlation to x.obs and y that isn't in w.
 ## in other words, the model is missing an important feature of x.obs that's related to y.
-simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
+simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, y_bias=-0.8,z_bias=0,Px=0.5,accuracy_imbalance_difference=0.3){
     set.seed(seed)
     # make w and y dependent
-    z <- rbinom(N, 1, 0.5)
+    z <- rnorm(N,sd=0.5)
-    x <- rbinom(N, 1, Bzx * z + 0.5)
+    x <- rbinom(N, 1, plogis(Bzx * z + qlogis(Px)))
 
     y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
     y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
@@ -49,109 +49,122 @@ simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.
         df <- df[, x.obs := x]
     }
 
-    ## px <- mean(x)
+    ## probablity of an error is correlated with y
+    ## pz <- mean(z)
     ## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
 
     ## # this works because of conditional probability
-    ## accuracy_x0 <- prediction_accuracy / (px*(accuracy_imbalance_ratio) + (1-px))
+    ## accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
-    ## accuracy_x1 <- accuracy_imbalance_ratio * accuracy_x0
+    ## accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
 
-    ## x0 <- df[x==0]$x
+    ## z0x0 <- df[(z==0) & (x==0)]$x
-    ## x1 <- df[x==1]$x
+    ## z0x1 <- df[(z==0) & (x==1)]$x
-    ## nx1 <- nrow(df[x==1])
+    ## z1x0 <- df[(z==1) & (x==0)]$x
-    ## nx0 <- nrow(df[x==0])
+    ## z1x1 <- df[(z==1) & (x==1)]$x
 
-    ## yx0 <- df[x==0]$y
+    ## yz0x0 <- df[(z==0) & (x==0)]$y
-    ## yx1 <- df[x==1]$y 
+    ## yz0x1 <- df[(z==0) & (x==1)]$y
+    ## yz1x0 <- df[(z==1) & (x==0)]$y
+    ## yz1x1 <- df[(z==1) & (x==1)]$y
 
-    # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+    ## nz0x0 <- nrow(df[(z==0) & (x==0)])
-    ## acc.x0 <- plogis(0.5*scale(yx0) + qlogis(accuracy_x0))
+    ## nz0x1 <- nrow(df[(z==0) & (x==1)])
-    ## acc.x1 <- plogis(1.5*scale(yx1) + qlogis(accuracy_x1))
+    ## nz1x0 <- nrow(df[(z==1) & (x==0)])
+    ## nz1x1 <- nrow(df[(z==1) & (x==1)])
 
-    ## w0x0 <- (1-x0)**2 + (-1)**(1-x0) * acc.x0
+    ## yz1 <- df[z==1]$y 
-    ## w0x1 <- (1-x1)**2 + (-1)**(1-x1) * acc.x1
+    ## yz1 <- df[z==1]$y 
-    pz <- mean(z)
-    accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
 
-    # this works because of conditional probability
+    ## # tranform yz0.1 into a logistic distribution with mean accuracy_z0
-    accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+    ## acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
-    accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
+    ## acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
+    ## acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
+    ## acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
 
-    z0x0 <- df[(z==0) & (x==0)]$x
+    ## w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
-    z0x1 <- df[(z==0) & (x==1)]$x
+    ## w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
-    z1x0 <- df[(z==1) & (x==0)]$x
+    ## w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
-    z1x1 <- df[(z==1) & (x==1)]$x
+    ## w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
 
-    yz0x0 <- df[(z==0) & (x==0)]$y
+    ## ##perrorz0 <- w0z0*(pyz0)
-    yz0x1 <- df[(z==0) & (x==1)]$y
+    ## ##perrorz1 <- w0z1*(pyz1)
-    yz1x0 <- df[(z==1) & (x==0)]$y
-    yz1x1 <- df[(z==1) & (x==1)]$y
 
-    nz0x0 <- nrow(df[(z==0) & (x==0)])
+    ## w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
-    nz0x1 <- nrow(df[(z==0) & (x==1)])
+    ## w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
-    nz1x0 <- nrow(df[(z==1) & (x==0)])
+    ## w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
-    nz1x1 <- nrow(df[(z==1) & (x==1)])
+    ## w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
 
-    yz1 <- df[z==1]$y 
+    ## df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
-    yz1 <- df[z==1]$y 
+    ## df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]    
+    ## df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]    
+    ## df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]    
 
-    # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+    ## df[,w_pred:=as.integer(w > 0.5)]
-    acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
+    ## print(mean(df[z==0]$x == df[z==0]$w_pred))
-    acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
+    ## print(mean(df[z==1]$x == df[z==1]$w_pred))
-    acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
+    ## print(mean(df$w_pred == df$x))
-    acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
 
-    w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
-    w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
-    w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
-    w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
 
-    ##perrorz0 <- w0z0*(pyz0)
+    resids <- resid(lm(y~x + z))
-    ##perrorz1 <- w0z1*(pyz1)
+    odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(resids[x==1])) + z_bias * qlogis(pnorm(z[x==1],sd(z)))
+    odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(resids[x==0])) + z_bias * qlogis(pnorm(z[x==0],sd(z)))
 
-    w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
+    ## acc.x0 <- p.correct[df[,x==0]]
-    w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
+    ## acc.x1 <- p.correct[df[,x==1]]
-    w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
+
-    w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
+    df[x==0,w:=plogis(rlogis(.N,odds.x0))]
+    df[x==1,w:=plogis(rlogis(.N,odds.x1))]
+
+    df[,w_pred := as.integer(w > 0.5)]
 
-    df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
-    df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]    
-    df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]    
-    df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]    
 
-    df[,w_pred:=as.integer(w > 0.5)]
-    print(mean(df[z==0]$x == df[z==0]$w_pred))
-    print(mean(df[z==1]$x == df[z==1]$w_pred))
     print(mean(df$w_pred == df$x))
+    print(mean(df[y>=0]$w_pred == df[y>=0]$x))
+    print(mean(df[y<=0]$w_pred == df[y<=0]$x))
     return(df)
 }
 
 parser <- arg_parser("Simulate data and fit corrected models")
-parser <- add_argument(parser, "--N", default=1400, help="number of observations of w")
+parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
 parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
-parser <- add_argument(parser, "--seed", default=50, help='seed for the rng')
+parser <- add_argument(parser, "--seed", default=51, help='seed for the rng')
 parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
-parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01)
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
-parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.75)
 parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
 parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
 parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
-
+parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
-
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*z*x")
+parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-0.5)
+parser <- add_argument(parser, "--z_bias", help='coefficient of z on the probability a classification is correct', default=0)
+parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
+parser <- add_argument(parser, "--Px", help='base rate of x', default=0.5)
+parser <- add_argument(parser, "--confint_method", help='method for approximating confidence intervals', default='quad')
 args <- parse_args(parser)
 
 B0 <- 0
-Bxy <- 0.3
+Px <- args$Px
+Bxy <- args$Bxy
 Bzy <- args$Bzy
+Bzx <- args$Bzx
 
 if(args$m < args$N){
-    df <- simulate_data(args$N, args$m, B0, Bxy, args$Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)
 
-    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, error='')
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, y_bias=args$y_bias)
 
-    outline <- run_simulation(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x+z+y+x:y, truth_formula=x~z)
+    ## df.pc <- df[,.(x,y,z,w_pred,w)]
+    ##                                     #    df.pc <- df.pc[,err:=x-w_pred]
+    ## pc.df <- pc(suffStat=list(C=cor(df.pc),n=nrow(df.pc)),indepTest=gaussCItest,labels=names(df.pc),alpha=0.05)
+    ## plot(pc.df)
 
-    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, 'Bzx'=args$Bzx, 'Bzy'=Bzy, 'Px'=Px, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'y_bias'=args$y_bias,'outcome_formula'=args$outcome_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, confint_method=args$confint_method, error='')
+
+    outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula),confint_method=args$confint_method)
+    
+   
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
     if(file.exists(args$outfile)){
         logdata <- read_feather(args$outfile)
         logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)

simulations/03_depvar.R

@@ -0,0 +1,111 @@
+### EXAMPLE 1: demonstrates how measurement error can lead to a type sign error in a covariate
+### What kind of data invalidates fong + tyler?
+### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
+### Even when you include the proxy variable in the regression.
+### But with some ground truth and multiple imputation, you can fix it.
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+library(predictionError)
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+setDTthreads(40)
+
+source("simulation_base.R")
+
+## SETUP:
+### we want to estimate x -> y; x is MAR
+### we have x -> k; k -> w; x -> w is used to predict x via the model w.
+### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
+### The labels x are binary, but the model provides a continuous predictor
+
+### simulation:
+#### how much power do we get from the model in the first place? (sweeping N and m)
+#### 
+
+## one way to do it is by adding correlation to x.obs and y that isn't in w.
+## in other words, the model is missing an important feature of x.obs that's related to y.
+simulate_data <- function(N, m, B0, Bxy, Bzy, Bzx, seed, prediction_accuracy=0.73, log.likelihood.gain = 0.1){
+    set.seed(seed)
+    set.seed(seed)
+
+    # make w and y dependent
+    z <- rnorm(N, sd=0.5)
+    x <- rbinom(N, 1, plogis(Bzx*z))
+
+    ystar <- Bzy * z + Bxy * x + B0
+    y <- rbinom(N,1,plogis(ystar))
+
+    df <- data.table(x=x,y=y,ystar=ystar,z=z)
+
+    if(m < N){
+        df <- df[sample(nrow(df), m), y.obs := y]
+    } else {
+        df <- df[, y.obs := y]
+    }
+    
+    odds.y1 <- qlogis(prediction_accuracy)
+    odds.y0 <- qlogis(prediction_accuracy,lower.tail=F)
+
+    df[y==0,w:=plogis(rlogis(.N,odds.y0))]
+    df[y==1,w:=plogis(rlogis(.N,odds.y1))]
+
+    df[,w_pred := as.integer(w > 0.5)]
+
+    print(mean(df[x==0]$y == df[x==0]$w_pred))
+    print(mean(df[x==1]$y == df[x==1]$w_pred))
+    print(mean(df$w_pred == df$y))
+    return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=10000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.72)
+## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
+## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
+parser <- add_argument(parser, "--Bzx", help='coeffficient of z on x', default=-0.5)
+parser <- add_argument(parser, "--B0", help='Base rate of y', default=0.5)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y")
+parser <- add_argument(parser, "--confint_method", help='method for getting confidence intervals', default="quad")
+
+args <- parse_args(parser)
+
+B0 <- args$B0
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+Bzx <- args$Bzx
+
+if(args$m < args$N){
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, args$seed, args$prediction_accuracy)
+
+#    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx,'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula, 'confint_method'=args$confint_method)
+
+    outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula), confint_method=args$confint_method)
+
+    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }
+
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

simulations/03_depvar_differential.R

@@ -31,13 +31,14 @@ source("simulation_base.R")
 
 ## one way to do it is by adding correlation to x.obs and y that isn't in w.
 ## in other words, the model is missing an important feature of x.obs that's related to y.
-simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
+simulate_data <- function(N, m, B0, Bxy, Bzy, Py, seed, prediction_accuracy=0.73, x_bias=-0.75){
     set.seed(seed)
+
     # make w and y dependent
     z <- rbinom(N, 1, 0.5)
     x <- rbinom(N, 1, 0.5)
 
-    ystar <- Bzy * z + Bxy * x
+    ystar <- Bzy * z + Bxy * x + B0 + qlogix(Py)
     y <- rbinom(N,1,plogis(ystar))
 
     # glm(y ~ x + z, family="binomial")
@@ -50,39 +51,17 @@ simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, ac
         df <- df[, y.obs := y]
     }
     
-    df <- df[,w_pred:=y]
+    odds.y1 <- qlogis(prediction_accuracy) + x_bias*df[y==1]$x
+    odds.y0 <- qlogis(prediction_accuracy,lower.tail=F) + x_bias*df[y==0]$x
 
-    pz <- mean(z)
+    df[y==0,w:=plogis(rlogis(.N,odds.y0))]
+    df[y==1,w:=plogis(rlogis(.N,odds.y1))]
 
-    accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
+    df[,w_pred := as.integer(w > 0.5)]
 
-    # this works because of conditional probability
+    print(mean(df[x==0]$y == df[x==0]$w_pred))
-    accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+    print(mean(df[x==1]$y == df[x==1]$w_pred))
-    accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
-
-
-    yz0 <- df[z==0]$y
-    yz1 <- df[z==1]$y
-    nz1 <- nrow(df[z==1])
-    nz0 <- nrow(df[z==0])
-
-    acc_z0 <- plogis(0.7*scale(yz0) + qlogis(accuracy_z0))
-    acc_z1 <- plogis(1.3*scale(yz1) + qlogis(accuracy_z1))
-    
-    w0z0 <- (1-yz0)**2 + (-1)**(1-yz0) * acc_z0
-    w0z1 <- (1-yz1)**2 + (-1)**(1-yz1) * acc_z1
-    
-    w0z0.noisy.odds <- rlogis(nz0,qlogis(w0z0))
-    w0z1.noisy.odds <- rlogis(nz1,qlogis(w0z1))
-    df[z==0,w:=plogis(w0z0.noisy.odds)]
-    df[z==1,w:=plogis(w0z1.noisy.odds)]
-
-    df[,w_pred:=as.integer(w > 0.5)]
-
-    print(mean(df[y==0]$y == df[y==0]$w_pred))
-    print(mean(df[y==1]$y == df[y==1]$w_pred))
     print(mean(df$w_pred == df$y))
-
     return(df)
 }
 
@@ -92,21 +71,30 @@ parser <- add_argument(parser, "--m", default=500, help="m the number of ground
 parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
 parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
 parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
-parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.8)
-parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
+## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
+## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
+parser <- add_argument(parser, "--x_bias", help='how is the classifier biased?', default=0.75)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
+parser <- add_argument(parser, "--Py", help='Base rate of y', default=0.5)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*x")
 
 args <- parse_args(parser)
 
 B0 <- 0
-Bxy <- 0.7
+Bxy <- args$Bxy
-Bzy <- -0.7
+Bzy <- args$Bzy
+
 
 if(args$m < args$N){
-    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$x_bias_y0, args$x_bias_y1)
 
-    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
+#    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias'=args$x_bias,'x_bias'=args$x_bias,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
 
-    outline <- run_simulation_depvar(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ y*x + y*z + z*x)
+    outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula))
 
     outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
 

simulations/03_indep_differential_nonnorm.R

@@ -0,0 +1,185 @@
+### EXAMPLE 1: demonstrates how measurement error can lead to a type sign error in a covariate
+### What kind of data invalidates fong + tyler?
+### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
+### Even when you include the proxy variable in the regression.
+### But with some ground truth and multiple imputation, you can fix it.
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+library(predictionError)
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+setDTthreads(40)
+
+source("simulation_base.R")
+
+## SETUP:
+### we want to estimate x -> y; x is MAR
+### we have x -> k; k -> w; x -> w is used to predict x via the model w.
+### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
+### The labels x are binary, but the model provides a continuous predictor
+
+### simulation:
+#### how much power do we get from the model in the first place? (sweeping N and m)
+#### 
+
+## one way to do it is by adding correlation to x.obs and y that isn't in w.
+## in other words, the model is missing an important feature of x.obs that's related to y.
+simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, y_bias=-0.8,z_bias=0,Px=0.5,accuracy_imbalance_difference=0.3,sd_y_mixin=1){
+    set.seed(seed)
+    # make w and y dependent
+    z <- rnorm(N,sd=0.5)
+    x <- rbinom(N, 1, plogis(Bzx * z + qlogis(Px)))
+    ## following Fong + Tyler: mix y with a Bernoulli(0.15) × |N (0, 20)| to make a skewed non-normal distribution
+    y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
+    y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
+    y <- Bzy * z + Bxy * x + y.epsilon + rbinom(N,1,0.15) * rnorm(N,0,sd_y_mixin)
+    
+    df <- data.table(x=x,y=y,z=z)
+
+    if(m < N){
+        df <- df[sample(nrow(df), m), x.obs := x]
+    } else {
+        df <- df[, x.obs := x]
+    }
+
+    ## probablity of an error is correlated with y
+    ## pz <- mean(z)
+    ## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
+
+    ## # this works because of conditional probability
+    ## accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
+    ## accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
+
+    ## z0x0 <- df[(z==0) & (x==0)]$x
+    ## z0x1 <- df[(z==0) & (x==1)]$x
+    ## z1x0 <- df[(z==1) & (x==0)]$x
+    ## z1x1 <- df[(z==1) & (x==1)]$x
+
+    ## yz0x0 <- df[(z==0) & (x==0)]$y
+    ## yz0x1 <- df[(z==0) & (x==1)]$y
+    ## yz1x0 <- df[(z==1) & (x==0)]$y
+    ## yz1x1 <- df[(z==1) & (x==1)]$y
+
+    ## nz0x0 <- nrow(df[(z==0) & (x==0)])
+    ## nz0x1 <- nrow(df[(z==0) & (x==1)])
+    ## nz1x0 <- nrow(df[(z==1) & (x==0)])
+    ## nz1x1 <- nrow(df[(z==1) & (x==1)])
+
+    ## yz1 <- df[z==1]$y 
+    ## yz1 <- df[z==1]$y 
+
+    ## # tranform yz0.1 into a logistic distribution with mean accuracy_z0
+    ## acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
+    ## acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
+    ## acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
+    ## acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
+
+    ## w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
+    ## w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
+    ## w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
+    ## w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
+
+    ## ##perrorz0 <- w0z0*(pyz0)
+    ## ##perrorz1 <- w0z1*(pyz1)
+
+    ## w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
+    ## w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
+    ## w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
+    ## w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
+
+    ## df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
+    ## df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]    
+    ## df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]    
+    ## df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]    
+
+    ## df[,w_pred:=as.integer(w > 0.5)]
+    ## print(mean(df[z==0]$x == df[z==0]$w_pred))
+    ## print(mean(df[z==1]$x == df[z==1]$w_pred))
+    ## print(mean(df$w_pred == df$x))
+
+
+
+    resids <- resid(lm(y~x + z))
+    odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(resids[x==1],log.p=T),log.p=T) + z_bias * qlogis(pnorm(z[x==1],sd(z),log.p=T),log.p=T)
+    odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(resids[x==0],log.p=T),log.p=T) + z_bias * qlogis(pnorm(z[x==0],sd(z),log.p=T),log.p=T)
+
+    ## acc.x0 <- p.correct[df[,x==0]]
+    ## acc.x1 <- p.correct[df[,x==1]]
+
+    df[x==0,w:=plogis(rlogis(.N,odds.x0))]
+    df[x==1,w:=plogis(rlogis(.N,odds.x1))]
+
+    print(prediction_accuracy)
+    print(resids[is.na(df$w)])
+    print(odds.x0[is.na(df$w)])
+    print(odds.x1[is.na(df$w)])
+
+    df[,w_pred := as.integer(w > 0.5)]
+
+
+    print(mean(df$w_pred == df$x))
+    print(mean(df[y>=0]$w_pred == df[y>=0]$x))
+    print(mean(df[y<=0]$w_pred == df[y<=0]$x))
+    return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=51, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.75)
+parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
+parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*z*x")
+parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-0.5)
+parser <- add_argument(parser, "--z_bias", help='coefficient of z on the probability a classification is correct', default=0)
+parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
+parser <- add_argument(parser, "--Px", help='base rate of x', default=0.5)
+parser <- add_argument(parser, "--confint_method", help='method for approximating confidence intervals', default='quad')
+parser <- add_argument(parser, "--sd_y_mixin", help='varience of the non-normal part of Y', default=10)
+args <- parse_args(parser)
+
+B0 <- 0
+Px <- args$Px
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+Bzx <- args$Bzx
+
+if(args$m < args$N){
+
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, y_bias=args$y_bias, sd_y_mixin=args$sd_y_mixin)
+
+    ## df.pc <- df[,.(x,y,z,w_pred,w)]
+    ##                                     #    df.pc <- df.pc[,err:=x-w_pred]
+    ## pc.df <- pc(suffStat=list(C=cor(df.pc),n=nrow(df.pc)),indepTest=gaussCItest,labels=names(df.pc),alpha=0.05)
+    ## plot(pc.df)
+
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, 'Bzx'=args$Bzx, 'Bzy'=Bzy, 'Px'=Px, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'y_bias'=args$y_bias,'outcome_formula'=args$outcome_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, confint_method=args$confint_method, error='', 'sd_y_mixin'=args$sd_y_mixin)
+
+    outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula),confint_method=args$confint_method)
+    
+   
+ outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }
+
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

simulations/04_depvar_differential.R

@@ -0,0 +1,112 @@
+### EXAMPLE 1: demonstrates how measurement error can lead to a type sign error in a covariate
+### What kind of data invalidates fong + tyler?
+### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
+### Even when you include the proxy variable in the regression.
+### But with some ground truth and multiple imputation, you can fix it.
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+library(predictionError)
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+setDTthreads(40)
+
+source("simulation_base.R")
+
+## SETUP:
+### we want to estimate x -> y; x is MAR
+### we have x -> k; k -> w; x -> w is used to predict x via the model w.
+### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
+### The labels x are binary, but the model provides a continuous predictor
+
+### simulation:
+#### how much power do we get from the model in the first place? (sweeping N and m)
+#### 
+
+## one way to do it is by adding correlation to x.obs and y that isn't in w.
+## in other words, the model is missing an important feature of x.obs that's related to y.
+simulate_data <- function(N, m, B0, Bxy, Bzy, Bxz=0, seed=0, prediction_accuracy=0.73, z_bias=-0.75){
+    set.seed(seed)
+
+    # make w and y dependent
+    z <- rnorm(N,sd=0.5)
+    x <- rbinom(N,1,plogis(Bxz*z))
+
+    ystar <- Bzy * z + Bxy * x + B0
+    y <- rbinom(N,1,plogis(ystar))
+
+    # glm(y ~ x + z, family="binomial")
+
+    df <- data.table(x=x,y=y,ystar=ystar,z=z)
+
+    if(m < N){
+        df <- df[sample(nrow(df), m), y.obs := y]
+    } else {
+        df <- df[, y.obs := y]
+    }
+    
+    odds.y1 <- qlogis(prediction_accuracy) + z_bias*df[y==1]$z
+    odds.y0 <- qlogis(prediction_accuracy,lower.tail=F) + z_bias*df[y==0]$z
+
+    df[y==0,w:=plogis(rlogis(.N,odds.y0))]
+    df[y==1,w:=plogis(rlogis(.N,odds.y1))]
+
+    df[,w_pred := as.integer(w > 0.5)]
+
+    print(mean(df[x==0]$y == df[x==0]$w_pred))
+    print(mean(df[x==1]$y == df[x==1]$w_pred))
+    print(mean(df$w_pred == df$y))
+    return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_4.feather')
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.75)
+## parser <- add_argument(parser, "--z_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
+## parser <- add_argument(parser, "--z_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
+parser <- add_argument(parser, "--z_bias", help='how is the classifier biased?', default=-0.5)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.7)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.7)
+parser <- add_argument(parser, "--Bzx", help='coeffficient of z on y', default=1)
+parser <- add_argument(parser, "--B0", help='coeffficient of z on y', default=0)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y+z")
+parser <- add_argument(parser, "--confint_method", help='method for approximating confidence intervals', default='quad')
+args <- parse_args(parser)
+
+B0 <- args$B0
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+Bzx <- args$Bzx
+
+if(args$m < args$N){
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzx, Bzy, args$seed, args$prediction_accuracy, args$z_bias)
+
+#    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy,'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'z_bias_y0'=args$z_bias_y0,'z_bias_y1'=args$z_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy,'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'z_bias'=args$z_bias,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula, confint_method=args$confint_method)
+
+    outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula),confint_method=args$confint_method)
+    print(outline$error.cor.z)
+
+    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }
+
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

simulations/05_irr_indep.R

@@ -0,0 +1,116 @@
+### EXAMPLE 2_b: demonstrates how measurement error can lead to a type
+### sign error in a covariate This is the same as example 2, only
+### instead of x->k we have k->x.  Even when you have a good
+### predictor, if it's biased against a covariate you can get the
+### wrong sign.  Even when you include the proxy variable in the
+### regression.  But with some ground truth and multiple imputation,
+### you can fix it.
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+
+library(predictionError)
+options(amelia.parallel="no", amelia.ncpus=1)
+
+source("irr_simulation_base.R")
+
+## SETUP:
+### we want to estimate x -> y; x is MAR
+### we have x -> k; k -> w; x -> w is used to predict x via the model w.
+### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
+### The labels x are binary, but the model provides a continuous predictor
+
+### simulation:
+#### how much power do we get from the model in the first place? (sweeping N and m)
+#### 
+
+simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, coder_accuracy=0.9, seed=1){
+    set.seed(seed)
+    z <- rbinom(N, 1, 0.5)
+                                        #    x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance)
+    xprime <- Bzx * z #+ x.var.epsilon
+    x <- rbinom(N,1,plogis(xprime))
+
+    y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance)
+    y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
+    y <- Bzy * z + Bxy * x + y.epsilon + B0
+
+    df <- data.table(x=x,y=y,z=z)
+
+    if(m < N){
+        df <- df[sample(nrow(df), m), x.obs := x]
+    } else {
+        df <- df[, x.obs := x]
+    }
+
+    coder.0.correct <- rbinom(m, 1, coder_accuracy)
+    coder.1.correct <- rbinom(m, 1, coder_accuracy)
+
+    df[!is.na(x.obs),x.obs.0 := as.numeric((x.obs & coder.0.correct) | (!x.obs & !coder.0.correct))]
+    df[!is.na(x.obs),x.obs.1 := as.numeric((x.obs & coder.1.correct) | (!x.obs & !coder.1.correct))]
+
+
+    ## how can you make a model with a specific accuracy?
+    w0 =(1-x)**2 + (-1)**(1-x) * prediction_accuracy
+
+    ## how can you make a model with a specific accuracy, with a continuous latent variable.
+    # now it makes the same amount of mistake to each point, probably
+    # add mean0 noise to the odds.
+    
+    w.noisey.odds = rlogis(N,qlogis(w0))
+    df[,w := plogis(w.noisey.odds)]
+    df[,w_pred:=as.integer(w > 0.5)]
+    (mean(df$x==df$w_pred))
+    return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=150, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=1, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
+# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
+parser <- add_argument(parser, "--coder_accuracy", help='how accurate are the human coders?', default=0.85)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~x")
+
+# parser <- add_argument(parser, "--rater_formula", help='formula for the true variable', default="x.obs~x")
+parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
+parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=-0.3)
+parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=0.27)
+parser <- add_argument(parser, "--Bxy", help='Effect of x on y', default=-0.33)
+
+args <- parse_args(parser)
+B0 <- 0
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+Bzx <- args$Bzx
+
+if (args$m < args$N){
+
+    df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed, y_explained_variance = args$y_explained_variance,  prediction_accuracy=args$prediction_accuracy, coder_accuracy=args$coder_accuracy)
+
+    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'outcome_formula'=args$outcome_formula, 'truth_formula'=args$truth_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, 'coder_accuracy'=args$coder_accuracy, error='')
+
+    outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula))
+    
+    outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+    if(file.exists(args$outfile)){
+        logdata <- read_feather(args$outfile)
+        logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+    } else {
+        logdata <- as.data.table(outline)
+    }
+
+    print(outline)
+    write_feather(logdata, args$outfile)
+    unlock(outfile_lock)
+}

simulations/06_irr_dv.R

@@ -0,0 +1,101 @@
+
+library(argparser)
+library(mecor)
+library(ggplot2)
+library(data.table)
+library(filelock)
+library(arrow)
+library(Amelia)
+library(Zelig)
+library(predictionError)
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+setDTthreads(40)
+
+source("irr_dv_simulation_base.R")
+
+
+## one way to do it is by adding correlation to x.obs and y that isn't in w.
+## in other words, the model is missing an important feature of x.obs that's related to y.
+simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, coder_accuracy=0.8){
+    set.seed(seed)
+
+    # make w and y dependent
+    z <- rbinom(N, 1, 0.5)
+    x <- rbinom(N, 1, 0.5)
+
+    ystar <- Bzy * z + Bxy * x + B0
+    y <- rbinom(N,1,plogis(ystar))
+
+    # glm(y ~ x + z, family="binomial")
+
+    df <- data.table(x=x,y=y,ystar=ystar,z=z)
+
+    df <- df[sample(nrow(df), m), y.obs := y]
+    
+    coder.0.correct <- rbinom(m, 1, coder_accuracy)
+    coder.1.correct <- rbinom(m, 1, coder_accuracy)
+    
+    df[!is.na(y.obs),y.obs.0 := as.numeric((.SD$y.obs & coder.0.correct) | (!.SD$y.obs & !coder.0.correct))]
+    df[!is.na(y.obs),y.obs.1 := as.numeric((.SD$y.obs & coder.1.correct) | (!.SD$y.obs & !coder.1.correct))]
+
+    odds.y1 <- qlogis(prediction_accuracy)
+    odds.y0 <- qlogis(prediction_accuracy,lower.tail=F)
+
+    df[y==0,w:=plogis(rlogis(.N,odds.y0))]
+    df[y==1,w:=plogis(rlogis(.N,odds.y1))]
+
+    df[,w_pred := as.integer(w > 0.5)]
+
+    print(mean(df$y == df$y.obs.0,na.rm=T))
+    print(mean(df$y == df$y.obs.1,na.rm=T))
+
+    print(mean(df[x==0]$y == df[x==0]$w_pred))
+    print(mean(df[x==1]$y == df[x==1]$w_pred))
+    print(mean(df$w_pred == df$y))
+    return(df)
+}
+
+parser <- arg_parser("Simulate data and fit corrected models")
+parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
+parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
+parser <- add_argument(parser, "--seed", default=16, help='seed for the rng')
+parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
+parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
+parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
+## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
+## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
+parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
+parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
+parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
+parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y+y.obs.1+y.obs.0")
+parser <- add_argument(parser, "--coder_accuracy", help='How accurate are the coders?', default=0.8)
+
+args <- parse_args(parser)
+
+B0 <- 0
+Bxy <- args$Bxy
+Bzy <- args$Bzy
+
+
+df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$coder_accuracy)
+
+                                        #    result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
+
+outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula))
+
+outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
+
+if(file.exists(args$outfile)){
+    logdata <- read_feather(args$outfile)
+    logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
+} else {
+    logdata <- as.data.table(outline)
+}
+
+print(outline)
+write_feather(logdata, args$outfile)
+unlock(outfile_lock)
+
+warnings()

simulations/Makefile

@@ -1,20 +1,22 @@
-
+.ONESHELL:
 SHELL=bash
 
-Ns=[1000,3600,14400]
+Ns=[1000, 5000, 10000]
-ms=[75,150,300]
+ms=[100, 200, 400]
-seeds=[$(shell seq -s, 1 250)]
+seeds=[$(shell seq -s, 1 500)]
 explained_variances=[0.1]
 
-all:remembr.RDS
+all:main supplement
+main:remembr.RDS 
+supplement:robustness_1.RDS robustness_1_dv.RDS robustness_2.RDS robustness_2_dv.RDS robustness_3.RDS robustness_3_dv.RDS robustness_3_proflik.RDS robustness_3_dv_proflik.RDS robustness_4.RDS robustness_4_dv.RDS robustness_5.RDS robustness_5_dv.RDS robustness_6.feather
 
-srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1
+srun=sbatch --wait --verbose run_job.sbatch
 
 
 joblists:example_1_jobs example_2_jobs example_3_jobs
 
 # test_true_z_jobs: test_true_z.R simulation_base.R
-# 	grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
+# 	sbatch --wait --verbose run_job.sbatch  ./grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
 
 # test_true_z.feather: test_true_z_jobs 
 # 	rm -f test_true_z.feather
@@ -22,56 +24,504 @@ joblists:example_1_jobs example_2_jobs example_3_jobs
 # 	sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs
 
 
-example_1_jobs: 01_two_covariates.R simulation_base.R
+example_1_jobs: 01_two_covariates.R simulation_base.R grid_sweep.py pl_methods.R
-	grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs
+	${srun} ./grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[1]}' --outfile example_1_jobs
 
 example_1.feather: example_1_jobs 
 	rm -f example_1.feather
-	sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
+	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_1_jobs
-#	sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
+	sbatch --wait --verbose --array=1001-2000 run_simulation.sbatch 0 example_1_jobs
+	sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_1_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
 
-example_2_jobs: 02_indep_differential.R simulation_base.R
+example_2_jobs: 02_indep_differential.R simulation_base.R grid_sweep.py pl_methods.R
-	grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "accuracy_imbalance_difference":[0.3], "Bzy":[0.3]}' --outfile example_2_jobs
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+z+x"]}' --outfile example_2_jobs
 
 example_2.feather: example_2_jobs 
 	rm -f example_2.feather
-	sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
+	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_2_jobs
-#	sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
+	sbatch --wait --verbose --array=1001-2000 run_simulation.sbatch 0 example_2_jobs
+	sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_2_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
+
+
 
 # example_2_B_jobs: example_2_B.R
-# 	grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
+# 	sbatch --wait --verbose run_job.sbatch python3 ./grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
 
 # example_2_B.feather: example_2_B_jobs
 # 	rm -f example_2_B.feather
 # 	sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
 
-example_3_jobs: 03_depvar_differential.R simulation_base.R
+example_3_jobs: 03_depvar.R simulation_base.R grid_sweep.py pl_methods.R
-	grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "Bxy":[0.7],"Bzy":[-0.7],"Bzx":[1],"seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
 
 example_3.feather: example_3_jobs
 	rm -f example_3.feather	
-	sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l)  run_simulation.sbatch 0 example_3_jobs
+	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_3_jobs
+	sbatch --wait --verbose --array=1001-2000 run_simulation.sbatch 0 example_3_jobs
+	 sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_3_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs
+
+example_4_jobs: 04_depvar_differential.R simulation_base.R grid_sweep.py pl_methods.R
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"Bxy":[0.7],"Bzy":[-0.7],"Bzx":[1], "m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[-0.5], "prediction_accuracy":[0.73]}' --outfile example_4_jobs
+
+example_4.feather: example_4_jobs
+	rm -f example_4.feather
+	sbatch --wait --verbose --array=1-1000  run_simulation.sbatch 0 example_4_jobs
+	sbatch --wait --verbose --array=1001-2000  run_simulation.sbatch 0 example_4_jobs
+	sbatch --wait --verbose --array=2001-3001 run_simulation.sbatch 0 example_4_jobs
+	sbatch --wait --verbose --array=2001-3000  run_simulation.sbatch 0 example_4_jobs
+	sbatch --wait --verbose --array=3001-$(shell cat example_4_jobs | wc -l)  run_simulation.sbatch 0 example_4_jobs
 
 
-remembr.RDS:example_1.feather example_2.feather example_3.feather plot_example.R plot_dv_example.R
+remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R summarize_estimator.R
 	rm -f remembr.RDS
 	${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
 	${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
 	${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
+	${srun} Rscript plot_dv_example.R --infile example_4.feather --name "plot.df.example.4"
 
-clean:
+
+START=0
+STEP=1000
+ONE=1
+
+robustness_Ns=[1000,5000]
+robustness_ms=[100,200]
+
+#in robustness 1 / example 2 misclassification is correlated with Y.
+robustness_1_jobs_p1: 02_indep_differential.R simulation_base.R grid_sweep.py
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":[1000],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3,0],"Bxy":[0.3],"Bzx":[1,0], "outcome_formula":["y~x+z"], "z_bias":[0, 0.5], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs_p1
+
+robustness_1_jobs_p2: 02_indep_differential.R simulation_base.R grid_sweep.py
+	sbatch --wait --verbose run_job.sbatch ./grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":[5000],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3,0],"Bxy":[0.3],"Bzx":[1,0], "outcome_formula":["y~x+z"], "z_bias":[0, 0.5], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~1"]}' --outfile robustness_1_jobs_p2
+
+robustness_1.feather: robustness_1_jobs_p1 robustness_1_jobs_p2
+	rm -f $@
+	$(eval END_1!=cat robustness_1_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_1_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_1_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_1_jobs_p2;)
+
+robustness_1.RDS: robustness_1.feather summarize_estimator.R
+	rm -f robustness_1.RDS
+	${srun} Rscript plot_example.R --infile $< --name "robustness_1" --remember-file $@
+
+# when Bzy is 0 and zbias is not zero, we have the case where P(W|Y,X,Z) has an omitted variable that is conditionanlly independent from Y.  Note that X and Z are independent in this scenario.
+robustness_1_dv_jobs_p1: simulation_base.R 04_depvar_differential.R grid_sweep.py
+	${srun} ./grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict '{"N":[1000],"Bzx":[1], "Bxy":[0.7,0],"Bzy":[-0.7,0],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1_dv.feather"], "proxy_formula":["w_pred~y"],"z_bias":[-0.5]}' --outfile robustness_1_dv_jobs_p1
+
+robustness_1_dv_jobs_p2: simulation_base.R 04_depvar_differential.R grid_sweep.py
+	${srun} ./grid_sweep.py --command 'Rscript 04_depvar_differential.R' --arg_dict '{"N":[5000],"Bzx":[1], "Bxy":[0.7,0],"Bzy":[-0.7,0],"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_1_dv.feather"], "proxy_formula":["w_pred~y"],"z_bias":[-0.5]}' --outfile robustness_1_dv_jobs_p2
+
+robustness_1_dv.feather: robustness_1_dv_jobs_p1 robustness_1_dv_jobs_p2
+	rm -f $@
+	$(eval END_1!=cat robustness_1_dv_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_1_dv_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_1))
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_1_dv_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_1_dv_jobs_p2;)
+
+robustness_1_dv.RDS: robustness_1_dv.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript plot_dv_example.R --infile $< --name "robustness_1_dv" --remember-file $@
+
+
+robustness_2_jobs_p1: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.60,0.65]}' --outfile $@
+
+robustness_2_jobs_p2: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.70,0.75]}' --outfile $@
+
+robustness_2_jobs_p3: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.80,0.85]}' --outfile $@
+
+robustness_2_jobs_p4: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.90,0.95]}' --outfile $@
+
+robustness_2.feather: robustness_2_jobs_p1 robustness_2_jobs_p2 robustness_2_jobs_p3 robustness_2_jobs_p4
+	rm $@
+	$(eval END_1!=cat robustness_2_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_2_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_2_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_2_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_jobs_p4;)
+
+robustness_2.RDS: plot_example.R robustness_2.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_2" --remember-file $@
+
+robustness_2_dv_jobs_p1: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.60,0.65]}' --outfile $@
+
+robustness_2_dv_jobs_p2: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.70,0.75]}' --outfile $@
+
+robustness_2_dv_jobs_p3: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.80,0.85]}' --outfile $@
+
+robustness_2_dv_jobs_p4: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_2_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.90,0.95]}' --outfile $@
+
+robustness_2_dv.feather: robustness_2_dv_jobs_p1 robustness_2_dv_jobs_p2 robustness_2_dv_jobs_p3 robustness_2_dv_jobs_p4
+	rm -f $@
+	$(eval END_1!=cat robustness_2_dv_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_2_dv_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_2_dv_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_2_dv_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_dv_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_dv_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_dv_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_2_dv_jobs_p4;)
+
+robustness_2_dv.RDS: plot_dv_example.R robustness_2_dv.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_2_dv" --remember-file $@
+
+
+robustness_3_proflik_jobs: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":[1000],"m":[100], "seed":${seeds}, "outfile":["robustness_3_proflik.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.5,0.6,0.7,0.8,0.9,0.95], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85], "confint_method":['spline']}' --outfile $@
+
+robustness_3_proflik.feather: robustness_3_proflik_jobs
+	rm -f $@
+	$(eval END_1!=cat robustness_3_proflik_jobs | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_proflik_jobs;)
+
+robustness_3_proflik.RDS: plot_example.R robustness_3_proflik.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3_proflik" --remember-file $@
+
+
+robustness_3_jobs_p1: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.5,0.6], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
+
+robustness_3_jobs_p2: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.7,0.8], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
+
+robustness_3_jobs_p3: grid_sweep.py 01_two_covariates.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 01_two_covariates.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"Px":[0.9,0.95], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x"], "truth_formula":["x~z"], "prediction_accuracy":[0.85]}' --outfile $@
+
+robustness_3.feather: robustness_3_jobs_p1 robustness_3_jobs_p2 robustness_3_jobs_p3
+	rm -f $@
+	$(eval END_1!=cat robustness_3_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_3_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_3_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_jobs_p3;)
+
+robustness_3.RDS: plot_example.R robustness_3.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3" --remember-file $@
+
+robustness_3_dv_proflik_jobs: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":[1000],"m":[100], "seed":${seeds}, "outfile":["robustness_3_dv_proflik.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0,0.405,0.846,1.386,2.197,2.944], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85],"confint_method":['spline']}' --outfile $@
+
+robustness_3_dv_proflik.feather: robustness_3_dv_proflik_jobs
+	rm -f $@
+	$(eval END_1!=cat robustness_3_dv_proflik_jobs | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_dv_proflik_jobs;)
+
+robustness_3_dv_proflik.RDS: plot_dv_example.R robustness_3_dv_proflik.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3_dv_proflik" --remember-file $@
+
+
+ robustness_3_dv_jobs_p1: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0,0.405], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
+
+robustness_3_dv_jobs_p2: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1],"B0":[0.847,1.386], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
+
+
+robustness_3_dv_jobs_p3: grid_sweep.py 03_depvar.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_depvar.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_3_dv.feather"],"y_explained_variance":${explained_variances},  "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "B0":[2.197,2.944], "outcome_formula":["y~x+z"], "prediction_accuracy":[0.85]}' --outfile $@
+
+robustness_3_dv.feather: robustness_3_dv_jobs_p1 robustness_3_dv_jobs_p2 robustness_3_dv_jobs_p3
+	rm -f $@
+	$(eval END_1!=cat robustness_3_dv_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_3_dv_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_3_dv_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_dv_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_dv_jobs_p2;)
+	 $(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_3_dv_jobs_p3;)
+
+
+robustness_3_dv.RDS: plot_dv_example.R robustness_3_dv.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_3_dv" --remember-file $@
+
+
+
+robustness_4_jobs_p1: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[-2.944,-2.197]}' --outfile $@
+
+robustness_4_jobs_p2: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[-1.386,-0.846]}' --outfile $@
+
+robustness_4_jobs_p3: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f ./$@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[-0.405,-0.25]}' --outfile $@
+
+robustness_4_jobs_p4: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f ./$@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"], "y_bias":[0,-0.1]}' --outfile $@
+
+
+robustness_4.feather: robustness_4_jobs_p1 robustness_4_jobs_p2 robustness_4_jobs_p3
+	rm -f $@
+	$(eval END_1!=cat robustness_4_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_4_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_4_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_4_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_4))
+
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_jobs_p3;)
+
+
+robustness_4.RDS: plot_example.R robustness_4.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_4" --remember-file $@
+
+
+# '{"N":${robustness_Ns},"Bxy":[0.7],"Bzy":[-0.7],"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[0.5]}' --example_4_jobs
+
+robustness_4_dv_jobs_p1: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "z_bias":[0,0.1]}' --outfile $@
+
+robustness_4_dv_jobs_p2: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "z_bias":[0.25,0.405]}' --outfile $@
+
+robustness_4_dv_jobs_p3: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1],"outcome_formula":["y~x+z"],"z_bias":[0.846,1.386]}' --outfile $@ 
+
+robustness_4_dv_jobs_p4: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_4_dv.feather"],"Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1], "outcome_formula":["y~x+z"], "z_bias":[2.197,2.944]}' --outfile $@
+
+robustness_4_dv.feather: robustness_4_dv_jobs_p1 robustness_4_dv_jobs_p2 robustness_4_dv_jobs_p3 robustness_4_dv_jobs_p4
+	rm -f $@
+	$(eval END_1!=cat robustness_4_dv_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_4_dv_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_4_dv_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_4_dv_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_4_dv_jobs_p4;)
+
+
+robustness_4_dv.RDS: plot_dv_example.R robustness_4_dv.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_4_dv" --remember-file $@
+
+
+robustness_5_jobs_p1: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[1.386,2.197], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+robustness_5_jobs_p2: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.405,0.846], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+robustness_5_jobs_p3: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0,0.25], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+robustness_5_jobs_p4: grid_sweep.py 02_indep_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 02_indep_differential.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[2.944], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"]}' --outfile $@
+
+
+robustness_5.feather: robustness_5_jobs_p1 robustness_5_jobs_p2 robustness_5_jobs_p3
+	rm -f $@
+	$(eval END_1!=cat robustness_5_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_5_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_5_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_5_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_jobs_p4;)
+
+robustness_5.RDS: plot_example.R robustness_5.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_5" --remember-file $@
+
+
+# '{"N":${robustness_Ns},"Bxy":[0.7],"Bzy":[-0.7],"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "z_bias":[0.5]}' --example_4_jobs
+
+robustness_5_dv_jobs_p1: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	 ${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[0,0.25], "outcome_formula":["y~x+z"], "z_bias":[-0.5]}' --outfile $@
+
+robustness_5_dv_jobs_p2: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[0.405,0.846], "outcome_formula":["y~x+z"],  "z_bias":[-0.5]}' --outfile $@
+
+robustness_5_dv_jobs_p3: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"], "Bzy":[-0.7],"Bxy":[0.7],"Bzx":[1.386,2.197],"outcome_formula":["y~x+z"], "z_bias":[-0.5]}' --outfile $@ 
+
+robustness_5_dv_jobs_p4: grid_sweep.py 04_depvar_differential.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 04_depvar_differential.R'  --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_5_dv.feather"],"Bzy":[-0.7],"Bxy":[0.7],"Bzx":[2.944], "outcome_formula":["y~x+z"], "z_bias":[-0.5]}' --outfile $@
+
+robustness_5_dv.feather: robustness_5_dv_jobs_p1 robustness_5_dv_jobs_p2 robustness_5_dv_jobs_p3 robustness_5_dv_jobs_p4
+	rm -f $@
+	$(eval END_1!=cat robustness_5_dv_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_5_dv_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_5_dv_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_5_dv_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+
+	$(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p1;)
+	$(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p2;)
+	$(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_5_dv_jobs_p4;)
+
+
+robustness_5_dv.RDS: plot_dv_example.R robustness_5_dv.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_5_dv" --remember-file $@
+
+
+clean_main:
+	rm -f remembr.RDS
+	rm -f example_1_jobs
+	rm -f example_2_jobs
+	rm -f example_3_jobs
+	rm -f example_4_jobs
+	rm -f example_1.feather
+	rm -f example_2.feather
+	rm -f example_3.feather
+	rm -f example_4.feather
+
+
+#	
+clean_all:
 	rm *.feather
 	rm -f remembr.RDS
+	rm -f remembr*.RDS
+	rm -f robustness*.RDS
 	rm -f example_*_jobs
+	rm -f robustness_*_jobs_*
 #	sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs
 
 # example_2_B_mecor_jobs:
-# 	grid_sweep.py --command "Rscript example_2_B_mecor.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B_mecor.feather"]}' --outfile example_2_B_mecor_jobs
+# 	sbatch --wait --verbose run_job.sbatch grid_sweep.py --command "Rscript example_2_B_mecor.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B_mecor.feather"]}' --outfile example_2_B_mecor_jobs
 
 # example_2_B_mecor.feather:example_2_B_mecor.R example_2_B_mecor_jobs
 # 	rm -f example_2_B_mecor.feather
 # 	sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_mecor_jobs
 # 	sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_mecor_jobs
 
+robustness_6_jobs_p1: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[0,1,2.5]}' --outfile $@
+
+robustness_6_jobs_p2: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[5,10]}' --outfile $@
+
+robustness_6_jobs_p3: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[0,1,2.5],"y_bias":[0]}' --outfile $@
+
+robustness_6_jobs_p4: grid_sweep.py 03_indep_differential_nonnorm.R simulation_base.R
+	rm -f $@
+	${srun} ./$< --command 'Rscript 03_indep_differential_nonnorm.R' --arg_dict '{"N":${robustness_Ns},"m":${robustness_ms}, "seed":${seeds}, "outfile":["robustness_6.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y+x+z"], "truth_formula":["x~z"],"sd_y_mixin":[5,10],"y_bias":[0]}' --outfile $@
 
 
+robustness_6.feather: robustness_6_jobs_p1 robustness_6_jobs_p2 robustness_6_jobs_p3 robustness_6_jobs_p4
+	rm -f $@
+	$(eval END_1!=cat robustness_6_jobs_p1 | wc -l)
+	$(eval ITEMS_1!=seq $(START) $(STEP) $(END_1))
+	$(eval END_2!=cat robustness_6_jobs_p2 | wc -l)
+	$(eval ITEMS_2!=seq $(START) $(STEP) $(END_2))
+	$(eval END_3!=cat robustness_6_jobs_p3 | wc -l)
+	$(eval ITEMS_3!=seq $(START) $(STEP) $(END_3))
+	$(eval END_4!=cat robustness_6_jobs_p4 | wc -l)
+	$(eval ITEMS_4!=seq $(START) $(STEP) $(END_4))
+
+
+	# $(foreach item,$(ITEMS_1),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p1;)
+	# $(foreach item,$(ITEMS_2),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p2;)
+	# $(foreach item,$(ITEMS_3),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p3;)
+	$(foreach item,$(ITEMS_4),sbatch --wait --verbose --array=$(shell expr $(item) + $(ONE))-$(shell expr $(item) + $(STEP)) run_simulation.sbatch 0 robustness_6_jobs_p4;)
+
+
+robustness_6.RDS: plot_example.R robustness_6.feather summarize_estimator.R
+	rm -f $@
+	${srun} Rscript $< --infile $(word 2, $^) --name "robustness_6" --remember-file $@
+
+
+.PHONY: supplement

simulations/grid_sweep.py

@@ -2,8 +2,14 @@
 
 import fire
 from itertools import product
+import pyRemembeR
 
-def main(command, arg_dict, outfile):
+def main(command, arg_dict, outfile, remember_file='remember_grid_sweep.RDS'):
+    print(remember_file)
+    remember = pyRemembeR.remember.Remember()
+    remember.set_file(remember_file)
+    remember[outfile] = arg_dict
+    remember.save_to_r()
     keys = []
     values = []
         

simulations/irr_dv_simulation_base.R

@@ -0,0 +1,212 @@
+library(matrixStats) # for numerically stable logsumexps
+
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+library(Amelia)
+
+source("pl_methods.R")
+source("measerr_methods_2.R") ## for my more generic function.
+
+run_simulation_depvar <- function(df, result, outcome_formula = y ~ x + z, coder_formulas = list(y.obs.0 ~ 1, y.obs.1 ~ 1), proxy_formula = w_pred ~ y.obs.1+y.obs.0+y){
+    (accuracy <- df[,mean(w_pred==y)])
+    result <- append(result, list(accuracy=accuracy))
+    (error.cor.z <- cor(df$x, df$w_pred - df$z))
+    (error.cor.x <- cor(df$x, df$w_pred - df$y))
+    (error.cor.y <- cor(df$y, df$y - df$w_pred))
+    result <- append(result, list(error.cor.x = error.cor.x,
+                                  error.cor.z = error.cor.z,
+                                  error.cor.y = error.cor.y))
+
+    model.null <- glm(y~1, data=df,family=binomial(link='logit'))
+    (model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit')))
+    (lik.ratio <- exp(logLik(model.true) - logLik(model.null)))
+
+    true.ci.Bxy <- confint(model.true)['x',]
+    true.ci.Bzy <- confint(model.true)['z',]
+
+
+    result <- append(result, list(lik.ratio=lik.ratio))
+
+    result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
+                                  Bzy.est.true=coef(model.true)['z'],
+                                  Bxy.ci.upper.true = true.ci.Bxy[2],
+                                  Bxy.ci.lower.true = true.ci.Bxy[1],
+                                  Bzy.ci.upper.true = true.ci.Bzy[2],
+                                  Bzy.ci.lower.true = true.ci.Bzy[1]))
+                                  
+    (model.naive <- lm(y~w_pred+z, data=df))
+    
+    naive.ci.Bxy <- confint(model.naive)['w_pred',]
+    naive.ci.Bzy <- confint(model.naive)['z',]
+
+    result <- append(result, list(Bxy.est.naive=coef(model.naive)['w_pred'],
+                                  Bzy.est.naive=coef(model.naive)['z'],
+                                  Bxy.ci.upper.naive = naive.ci.Bxy[2],
+                                  Bxy.ci.lower.naive = naive.ci.Bxy[1],
+                                  Bzy.ci.upper.naive = naive.ci.Bzy[2],
+                                  Bzy.ci.lower.naive = naive.ci.Bzy[1]))
+
+
+
+    loa0.feasible <- glm(y.obs.0 ~ x + z, data = df[!(is.na(y.obs.0))], family=binomial(link='logit'))
+
+    loa0.ci.Bxy <- confint(loa0.feasible)['x',]
+    loa0.ci.Bzy <- confint(loa0.feasible)['z',]
+
+    result <- append(result, list(Bxy.est.loa0.feasible=coef(loa0.feasible)['x'],
+                                  Bzy.est.loa0.feasible=coef(loa0.feasible)['z'],
+                                  Bxy.ci.upper.loa0.feasible = loa0.ci.Bxy[2],
+                                  Bxy.ci.lower.loa0.feasible = loa0.ci.Bxy[1],
+                                  Bzy.ci.upper.loa0.feasible = loa0.ci.Bzy[2],
+                                  Bzy.ci.lower.loa0.feasible = loa0.ci.Bzy[1]))
+
+
+    ## df.loa0.mle <- copy(df)
+    ## df.loa0.mle[,y:=y.obs.0]
+    ## loa0.mle <- measerr_mle_dv(df.loa0.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+    ## fisher.info <- solve(loa0.mle$hessian)
+    ## coef <- loa0.mle$par
+    ## ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ## ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    ## result <- append(result, list(Bxy.est.loa0.mle=coef['x'],
+    ##                               Bzy.est.loa0.mle=coef['z'],
+    ##                               Bxy.ci.upper.loa0.mle = ci.upper['x'],
+    ##                               Bxy.ci.lower.loa0.mle = ci.lower['x'],
+    ##                               Bzy.ci.upper.loa0.mle = ci.upper['z'],
+    ##                               Bzy.ci.lower.loa0.mle = ci.upper['z']))
+
+    loco.feasible <- glm(y.obs.0 ~ x + z, data = df[(!is.na(y.obs.0)) & (y.obs.1 == y.obs.0)], family=binomial(link='logit'))
+
+    loco.feasible.ci.Bxy <- confint(loco.feasible)['x',]
+    loco.feasible.ci.Bzy <- confint(loco.feasible)['z',]
+
+    result <- append(result, list(Bxy.est.loco.feasible=coef(loco.feasible)['x'],
+                                  Bzy.est.loco.feasible=coef(loco.feasible)['z'],
+                                  Bxy.ci.upper.loco.feasible = loco.feasible.ci.Bxy[2],
+                                  Bxy.ci.lower.loco.feasible = loco.feasible.ci.Bxy[1],
+                                  Bzy.ci.upper.loco.feasible = loco.feasible.ci.Bzy[2],
+                                  Bzy.ci.lower.loco.feasible = loco.feasible.ci.Bzy[1]))
+
+    
+    ## df.double.proxy <- copy(df)
+    ## df.double.proxy <- df.double.proxy[,y.obs:=NA]
+    ## df.double.proxy <- df.double.proxy[,y:=NA]
+    
+    ## double.proxy.mle <- measerr_irr_mle_dv(df.double.proxy, outcome_formula=y~x+z, outcome_family=binomial(link='logit'), coder_formulas=list(y.obs.0 ~ y),  proxy_formula=w_pred ~ y.obs.0 + y, proxy_family=binomial(link='logit'))
+    ## print(double.proxy.mle$hessian)
+    ## fisher.info <- solve(double.proxy.mle$hessian)
+    ## coef <- double.proxy.mle$par
+    ## ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ## ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    ## result <- append(result, list(Bxy.est.double.proxy=coef['x'],
+    ##                               Bzy.est.double.proxy=coef['z'],
+    ##                               Bxy.ci.upper.double.proxy = ci.upper['x'],
+    ##                               Bxy.ci.lower.double.proxy = ci.lower['x'],
+    ##                               Bzy.ci.upper.double.proxy = ci.upper['z'],
+    ##                               Bzy.ci.lower.double.proxy = ci.lower['z']))
+
+
+    df.triple.proxy <- copy(df)
+    df.triple.proxy <- df.triple.proxy[,y.obs:=NA]
+    df.triple.proxy <- df.triple.proxy[,y:=NA]
+    
+    triple.proxy.mle <- measerr_irr_mle_dv(df.triple.proxy, outcome_formula=outcome_formula, outcome_family=binomial(link='logit'), coder_formulas=coder_formulas, proxy_formula=proxy_formula, proxy_family=binomial(link='logit'))
+    print(triple.proxy.mle$hessian)
+    fisher.info <- solve(triple.proxy.mle$hessian)
+    print(fisher.info)
+    coef <- triple.proxy.mle$par
+    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    result <- append(result, list(Bxy.est.triple.proxy=coef['x'],
+                                  Bzy.est.triple.proxy=coef['z'],
+                                  Bxy.ci.upper.triple.proxy = ci.upper['x'],
+                                  Bxy.ci.lower.triple.proxy = ci.lower['x'],
+                                  Bzy.ci.upper.triple.proxy = ci.upper['z'],
+                                  Bzy.ci.lower.triple.proxy = ci.lower['z']))
+
+    ## df.loco.mle <- copy(df)
+    ## df.loco.mle[,y.obs:=NA]
+    ## df.loco.mle[(y.obs.0)==(y.obs.1),y.obs:=y.obs.0]
+    ## df.loco.mle[,y.true:=y]
+    ## df.loco.mle[,y:=y.obs]
+    ## print(df.loco.mle[!is.na(y.obs.1),mean(y.true==y,na.rm=TRUE)])
+    ## loco.mle <- measerr_mle_dv(df.loco.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
+    ## fisher.info <- solve(loco.mle$hessian)
+    ## coef <- loco.mle$par
+    ## ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ## ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    ## result <- append(result, list(Bxy.est.loco.mle=coef['x'],
+    ##                               Bzy.est.loco.mle=coef['z'],
+    ##                               Bxy.ci.upper.loco.mle = ci.upper['x'],
+    ##                               Bxy.ci.lower.loco.mle = ci.lower['x'],
+    ##                               Bzy.ci.upper.loco.mle = ci.upper['z'],
+    ##                               Bzy.ci.lower.loco.mle = ci.lower['z']))
+
+
+
+    ## my implementatoin of liklihood based correction
+    mod.zhang <- zhang.mle.dv(df.loco.mle)
+    coef <- coef(mod.zhang)
+    ci <- confint(mod.zhang,method='quad')
+
+    result <- append(result,
+                     list(Bxy.est.zhang = coef['Bxy'],
+                          Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                          Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+                          Bzy.est.zhang = coef['Bzy'],
+                          Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+                          Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
+
+    
+
+    print(df.loco.mle)
+
+    # amelia says use normal distribution for binary variables.
+    tryCatch({
+        amelia.out.k <- amelia(df.loco.mle, m=200, p2s=0, idvars=c('y','ystar','w','y.obs.1','y.obs.0','y.true'))
+        mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+        (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+        est.x.mi <- coefse['x','Estimate']
+        est.x.se <- coefse['x','Std.Error']
+        result <- append(result,
+                         list(Bxy.est.amelia.full = est.x.mi,
+                              Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
+                              Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
+                              ))
+
+        est.z.mi <- coefse['z','Estimate']
+        est.z.se <- coefse['z','Std.Error']
+
+        result <- append(result,
+                         list(Bzy.est.amelia.full = est.z.mi,
+                              Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+                              Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
+                              ))
+
+    },
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <- paste0(result$error,'\n', e)
+    })
+    ## mle.irr <- measerr_irr_mle( df, outcome_formula = outcome_formula, rater_formula = rater_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+
+    ## fisher.info <- solve(mle.irr$hessian)
+    ## coef <- mle.irr$par
+    ## ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ## ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+    
+    ## result <- append(result,
+    ##                  list(Bxy.est.mle = coef['x'],
+    ##                       Bxy.ci.upper.mle = ci.upper['x'],
+    ##                       Bxy.ci.lower.mle = ci.lower['x'],
+    ##                       Bzy.est.mle = coef['z'],
+    ##                       Bzy.ci.upper.mle = ci.upper['z'],
+    ##                       Bzy.ci.lower.mle = ci.lower['z']))
+
+    return(result)
+
+}

simulations/irr_simulation_base.R

@@ -0,0 +1,222 @@
+library(matrixStats) # for numerically stable logsumexps
+
+options(amelia.parallel="no",
+        amelia.ncpus=1)
+library(Amelia)
+source("measerr_methods.R")
+source("pl_methods.R")
+
+run_simulation <- function(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ x, coder_formulas=list(x.obs.1~x, x.obs.0~x), truth_formula = x ~ z){
+
+    accuracy <- df[,mean(w_pred==x)]
+    result <- append(result, list(accuracy=accuracy))
+
+    (model.true <- lm(y ~ x + z, data=df))
+    true.ci.Bxy <- confint(model.true)['x',]
+    true.ci.Bzy <- confint(model.true)['z',]
+
+    result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
+                                  Bzy.est.true=coef(model.true)['z'],
+                                  Bxy.ci.upper.true = true.ci.Bxy[2],
+                                  Bxy.ci.lower.true = true.ci.Bxy[1],
+                                  Bzy.ci.upper.true = true.ci.Bzy[2],
+                                  Bzy.ci.lower.true = true.ci.Bzy[1]))
+
+
+
+
+
+    loa0.feasible <- lm(y ~ x.obs.0 + z, data = df[!(is.na(x.obs.1))])
+
+    loa0.ci.Bxy <- confint(loa0.feasible)['x.obs.0',]
+    loa0.ci.Bzy <- confint(loa0.feasible)['z',]
+
+    result <- append(result, list(Bxy.est.loa0.feasible=coef(loa0.feasible)['x.obs.0'],
+                                  Bzy.est.loa0.feasible=coef(loa0.feasible)['z'],
+                                  Bxy.ci.upper.loa0.feasible = loa0.ci.Bxy[2],
+                                  Bxy.ci.lower.loa0.feasible = loa0.ci.Bxy[1],
+                                  Bzy.ci.upper.loa0.feasible = loa0.ci.Bzy[2],
+                                  Bzy.ci.lower.loa0.feasible = loa0.ci.Bzy[1]))
+    print("fitting loa0 model")
+
+    df.loa0.mle <- copy(df)
+    df.loa0.mle[,x:=x.obs.0]
+    loa0.mle <- measerr_mle(df.loa0.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+    fisher.info <- solve(loa0.mle$hessian)
+    coef <- loa0.mle$par
+    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    result <- append(result, list(Bxy.est.loa0.mle=coef['x'],
+                                  Bzy.est.loa0.mle=coef['z'],
+                                  Bxy.ci.upper.loa0.mle = ci.upper['x'],
+                                  Bxy.ci.lower.loa0.mle = ci.lower['x'],
+                                  Bzy.ci.upper.loa0.mle = ci.upper['z'],
+                                  Bzy.ci.lower.loa0.mle = ci.upper['z']))
+
+
+
+    loco.feasible <- lm(y ~ x.obs.1 + z, data = df[(!is.na(x.obs.1)) & (x.obs.1 == x.obs.0)])
+
+
+    loco.feasible.ci.Bxy <- confint(loco.feasible)['x.obs.1',]
+    loco.feasible.ci.Bzy <- confint(loco.feasible)['z',]
+
+    result <- append(result, list(Bxy.est.loco.feasible=coef(loco.feasible)['x.obs.1'],
+                                  Bzy.est.loco.feasible=coef(loco.feasible)['z'],
+                                  Bxy.ci.upper.loco.feasible = loco.feasible.ci.Bxy[2],
+                                  Bxy.ci.lower.loco.feasible = loco.feasible.ci.Bxy[1],
+                                  Bzy.ci.upper.loco.feasible = loco.feasible.ci.Bzy[2],
+                                  Bzy.ci.lower.loco.feasible = loco.feasible.ci.Bzy[1]))
+
+
+    (model.naive <- lm(y~w_pred+z, data=df))
+    
+    naive.ci.Bxy <- confint(model.naive)['w_pred',]
+    naive.ci.Bzy <- confint(model.naive)['z',]
+
+    result <- append(result, list(Bxy.est.naive=coef(model.naive)['w_pred'],
+                                  Bzy.est.naive=coef(model.naive)['z'],
+                                  Bxy.ci.upper.naive = naive.ci.Bxy[2],
+                                  Bxy.ci.lower.naive = naive.ci.Bxy[1],
+                                  Bzy.ci.upper.naive = naive.ci.Bzy[2],
+                                  Bzy.ci.lower.naive = naive.ci.Bzy[1]))
+                                  
+    print("fitting loco model")
+
+    df.loco.mle <- copy(df)
+    df.loco.mle[,x.obs:=NA]
+    df.loco.mle[(x.obs.0)==(x.obs.1),x.obs:=x.obs.0]
+    df.loco.mle[,x.true:=x]
+    df.loco.mle[,x:=x.obs]
+    print(df.loco.mle[!is.na(x.obs.1),mean(x.true==x,na.rm=TRUE)])
+    loco.accuracy <- df.loco.mle[(!is.na(x.obs.1)) & (x.obs.1 == x.obs.0),mean(x.obs.1 == x.true)]    
+    loco.mle <- measerr_mle(df.loco.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+    fisher.info <- solve(loco.mle$hessian)
+    coef <- loco.mle$par
+    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    result <- append(result, list(loco.accuracy=loco.accuracy,
+                                  Bxy.est.loco.mle=coef['x'],
+                                  Bzy.est.loco.mle=coef['z'],
+                                  Bxy.ci.upper.loco.mle = ci.upper['x'],
+                                  Bxy.ci.lower.loco.mle = ci.lower['x'],
+                                  Bzy.ci.upper.loco.mle = ci.upper['z'],
+                                  Bzy.ci.lower.loco.mle = ci.lower['z']))
+
+    df.double.proxy.mle <- copy(df)
+    df.double.proxy.mle[,x.obs:=NA]
+    print("fitting double proxy model")
+
+    double.proxy.mle <- measerr_irr_mle(df.double.proxy.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula, coder_formulas=coder_formulas[1], truth_formula=truth_formula)
+    fisher.info <- solve(double.proxy.mle$hessian)
+    coef <- double.proxy.mle$par
+    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    result <- append(result, list(
+                                  Bxy.est.double.proxy=coef['x'],
+                                  Bzy.est.double.proxy=coef['z'],
+                                  Bxy.ci.upper.double.proxy = ci.upper['x'],
+                                  Bxy.ci.lower.double.proxy = ci.lower['x'],
+                                  Bzy.ci.upper.double.proxy = ci.upper['z'],
+                                  Bzy.ci.lower.double.proxy = ci.lower['z']))
+
+    df.triple.proxy.mle <- copy(df)
+    df.triple.proxy.mle[,x.obs:=NA]
+
+    print("fitting triple proxy model")
+    triple.proxy.mle <- measerr_irr_mle(df.triple.proxy.mle, outcome_formula=outcome_formula, proxy_formula=proxy_formula, coder_formulas=coder_formulas, truth_formula=truth_formula)
+    fisher.info <- solve(triple.proxy.mle$hessian)
+    coef <- triple.proxy.mle$par
+    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+
+    result <- append(result, list(
+                                  Bxy.est.triple.proxy=coef['x'],
+                                  Bzy.est.triple.proxy=coef['z'],
+                                  Bxy.ci.upper.triple.proxy = ci.upper['x'],
+                                  Bxy.ci.lower.triple.proxy = ci.lower['x'],
+                                  Bzy.ci.upper.triple.proxy = ci.upper['z'],
+                                  Bzy.ci.lower.triple.proxy = ci.lower['z']))
+    tryCatch({
+    amelia.out.k <- amelia(df.loco.mle, m=200, p2s=0, idvars=c('x.true','w','x.obs.1','x.obs.0','x'))
+    mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+    (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+
+    est.x.mi <- coefse['x.obs','Estimate']
+    est.x.se <- coefse['x.obs','Std.Error']
+    result <- append(result,
+                     list(Bxy.est.amelia.full = est.x.mi,
+                          Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
+                          Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
+                          ))
+
+    est.z.mi <- coefse['z','Estimate']
+    est.z.se <- coefse['z','Std.Error']
+
+    result <- append(result,
+                     list(Bzy.est.amelia.full = est.z.mi,
+                          Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+                          Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
+                          ))
+
+    },
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <-paste0(result$error,'\n', e)
+    }
+    )
+
+    tryCatch({
+
+        mod.zhang.lik <- zhang.mle.iv(df.loco.mle)
+        coef <- coef(mod.zhang.lik)
+        ci <- confint(mod.zhang.lik,method='quad')
+        result <- append(result,
+                         list(Bxy.est.zhang = coef['Bxy'],
+                              Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                              Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+                              Bzy.est.zhang = coef['Bzy'],
+                              Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+                              Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
+    },
+
+    error = function(e){
+        message("An error occurred:\n",e)
+        result$error <- paste0(result$error,'\n', e)
+    })
+
+    df <- df.loco.mle
+    N <- nrow(df)
+    m <- nrow(df[!is.na(x.obs)])
+    p <- v <- train <- rep(0,N)
+    M <- m
+    p[(M+1):(N)] <- 1
+    v[1:(M)] <- 1
+    df <- df[order(x.obs)]
+    y <- df[,y]
+    x <- df[,x.obs]
+    z <- df[,z]
+    w <- df[,w_pred]
+    # gmm gets pretty close
+    (gmm.res <- predicted_covariates(y, x, z, w, v, train, p, max_iter=100, verbose=TRUE))
+
+    result <- append(result,
+                     list(Bxy.est.gmm = gmm.res$beta[1,1],
+                          Bxy.ci.upper.gmm = gmm.res$confint[1,2],
+                          Bxy.ci.lower.gmm = gmm.res$confint[1,1],
+                          gmm.ER_pval = gmm.res$ER_pval
+                          ))
+
+    result <- append(result,
+                     list(Bzy.est.gmm = gmm.res$beta[2,1],
+                          Bzy.ci.upper.gmm = gmm.res$confint[2,2],
+                          Bzy.ci.lower.gmm = gmm.res$confint[2,1]))
+
+
+
+    return(result)
+
+}

simulations/measerr_methods.R

@@ -1,6 +1,7 @@
 library(formula.tools)
 library(matrixStats)
-
+library(optimx)
+library(bbmle)
 ## df: dataframe to model
 ## outcome_formula: formula for y | x, z
 ## outcome_family: family for y | x, z
@@ -14,18 +15,38 @@ library(matrixStats)
 
 ### ideal formulas for example 2
 # test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
-
+likelihood.logistic <- function(model.params, outcome, model.matrix){
+    ll <- vector(mode='numeric', length=length(outcome))
+    ll[outcome == 1] <- plogis(model.params %*% t(model.matrix[outcome==1,]), log=TRUE)
+    ll[outcome == 0] <- plogis(model.params %*% t(model.matrix[outcome==0,]), log=TRUE, lower.tail=FALSE)
+    return(ll)
+}
 
 ## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y 
-measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
+measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),maxit=1e6, method='optim',optim_method='L-BFGS-B'){
+    df.obs <- model.frame(outcome_formula, df)
+    proxy.model.matrix <- model.matrix(proxy_formula, df)
+    proxy.variable <- all.vars(proxy_formula)[1]
+
+    df.proxy.obs <- model.frame(proxy_formula,df)
+    proxy.obs <- with(df.proxy.obs, eval(parse(text=proxy.variable)))
+
+    response.var <- all.vars(outcome_formula)[1]
+    y.obs <- with(df.obs,eval(parse(text=response.var)))
+    outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
+
+    df.unobs <- df[is.na(df[[response.var]])]
+    df.unobs.y1 <- copy(df.unobs)
+    df.unobs.y1[[response.var]] <- 1
+    df.unobs.y0 <- copy(df.unobs)
+    df.unobs.y0[[response.var]] <- 0
+
+    outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
+    proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
+    proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
+    proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
 
     nll <- function(params){
-        df.obs <- model.frame(outcome_formula, df)
-        proxy.variable <- all.vars(proxy_formula)[1]
-        proxy.model.matrix <- model.matrix(proxy_formula, df)
-        response.var <- all.vars(outcome_formula)[1]
-        y.obs <- with(df.obs,eval(parse(text=response.var)))
-        outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
 
         param.idx <- 1
         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
@@ -38,12 +59,9 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
             ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
         }
 
-        df.obs <- model.frame(proxy_formula,df)
         n.proxy.model.covars <- dim(proxy.model.matrix)[2]
         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
-
         param.idx <- param.idx + n.proxy.model.covars
-        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
 
         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
             ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
@@ -53,14 +71,7 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
 
         ll.obs <- sum(ll.y.obs + ll.w.obs)
         
-        df.unobs <- df[is.na(df[[response.var]])]
-        df.unobs.y1 <- copy(df.unobs)
-        df.unobs.y1[[response.var]] <- 1
-        df.unobs.y0 <- copy(df.unobs)
-        df.unobs.y0[[response.var]] <- 1
-        
         ## integrate out y
-        outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
 
         if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
             ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
@@ -69,10 +80,6 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
             ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
         }
 
-        proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
-        proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
-        proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
-
         if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
             ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
             ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
@@ -98,23 +105,58 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
     start <- rep(0.1,length(params))
     names(start) <- params
     
-    fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    if(method=='optim'){
+        fit <- optim(start, fn = nll, lower=lower, method=optim_method, hessian=TRUE, control=list(maxit=maxit))
+    } else {
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+        print(quoted.names)
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+        measerr_mle_nll <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=maxit),method=optim_method)
+    }
     return(fit)
 }
 
-measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
 
-    measrr_mle_nll <- function(params){
+measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim', maxit=1e6, optim_method='L-BFGS-B'){
-        df.obs <- model.frame(outcome_formula, df)
 
-        proxy.variable <- all.vars(proxy_formula)[1]
+    df.obs <- model.frame(outcome_formula, df)
-        proxy.model.matrix <- model.matrix(proxy_formula, df)
+    response.var <- all.vars(outcome_formula)[1]
+    proxy.variable <- all.vars(proxy_formula)[1]
+    truth.variable <- all.vars(truth_formula)[1]
+    outcome.model.matrix <- model.matrix(outcome_formula, df)
+    proxy.model.matrix <- model.matrix(proxy_formula, df)
+    y.obs <- with(df.obs,eval(parse(text=response.var)))
 
-        response.var <- all.vars(outcome_formula)[1]
+    df.proxy.obs <- model.frame(proxy_formula,df)
-        y.obs <- with(df.obs,eval(parse(text=response.var)))
+    proxy.obs <- with(df.proxy.obs, eval(parse(text=proxy.variable)))
+    n.proxy.model.covars <- dim(proxy.model.matrix)[2]
 
-        outcome.model.matrix <- model.matrix(outcome_formula, df)
+    df.truth.obs <- model.frame(truth_formula, df)
+    truth.obs <- with(df.truth.obs, eval(parse(text=truth.variable)))
+    truth.model.matrix <- model.matrix(truth_formula,df.truth.obs)
+    n.truth.model.covars <- dim(truth.model.matrix)[2]
 
+    df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
+    df.unobs.x1 <- copy(df.unobs)
+    df.unobs.x1[,truth.variable] <- 1
+    df.unobs.x0 <- copy(df.unobs)
+    df.unobs.x0[,truth.variable] <- 0
+    outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
+            
+    outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
+    outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
+        
+    proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
+    proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
+    proxy.unobs <- df.unobs[[proxy.variable]]
+
+    truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs.x0)
+
+    measerr_mle_nll <- function(params){
         param.idx <- 1
         n.outcome.model.covars <- dim(outcome.model.matrix)[2]
         outcome.params <- params[param.idx:n.outcome.model.covars]
@@ -124,74 +166,50 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
         if(outcome_family$family == "gaussian"){
             sigma.y <- params[param.idx]
             param.idx <- param.idx + 1
+                                        #  outcome_formula likelihood using linear regression
             ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
-        }
+        } else if( (outcome_family$family == "binomial") & (outcome_family$link == "logit") )
+            ll.y.obs <- likelihood.logistic(outcome.params, y.obs, outcome.model.matrix)
+
     
-        df.obs <- model.frame(proxy_formula,df)
-        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
         proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
         param.idx <- param.idx + n.proxy.model.covars
-        proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
 
-        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
+        if( (proxy_family$family=="binomial") & (proxy_family$link=='logit'))
-            ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
+            ll.w.obs <- likelihood.logistic(proxy.params, proxy.obs, proxy.model.matrix)
-            ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
-            ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
-        }
-
-        df.obs <- model.frame(truth_formula, df)
-        truth.variable <- all.vars(truth_formula)[1]
-        truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
-        truth.model.matrix <- model.matrix(truth_formula,df)
-        n.truth.model.covars <- dim(truth.model.matrix)[2]
 
         truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
 
-        if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
+        if( (truth_family$family=="binomial") & (truth_family$link=='logit'))
-            ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
+            ll.x.obs <- likelihood.logistic(truth.params, truth.obs, truth.model.matrix)
-            ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
-            ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
-        }
         
+                                # add the three likelihoods
         ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
 
         ## likelihood for the predicted data
         ## integrate out the "truth" variable. 
         
         if(truth_family$family=='binomial'){
-            df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
-            df.unobs.x1 <- copy(df.unobs)
-            df.unobs.x1[,'x'] <- 1
-            df.unobs.x0 <- copy(df.unobs)
-            df.unobs.x0[,'x'] <- 0
-            outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
-            
-            outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
-            outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
             if(outcome_family$family=="gaussian"){
-            ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+                                        # likelihood of outcome
-            ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+                ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
+                ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
+            } else if( (outcome_family$family == "binomial") & (outcome_family$link == "logit") ){
+                ll.y.x1 <- likelihood.logistic(outcome.params, outcome.unobs, outcome.model.matrix.x1)
+                ll.y.x0 <- likelihood.logistic(outcome.params, outcome.unobs, outcome.model.matrix.x0)
             }
 
             if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
 
-                proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
+                ll.w.x0 <- likelihood.logistic(proxy.params, proxy.unobs, proxy.model.matrix.x0)
-                proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
+                ll.w.x1 <- likelihood.logistic(proxy.params, proxy.unobs, proxy.model.matrix.x1)
-                proxy.unobs <- df.unobs[[proxy.variable]]
-                ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
-                ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
 
-                ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
-                ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
-
-                ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
-                ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
             }
 
             if(truth_family$link=='logit'){
-                truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
+                                        # likelihood of truth
-                ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
+                ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
-                ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
+                ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
             }
         }
 
@@ -221,7 +239,355 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
     start <- rep(0.1,length(params))
     names(start) <- params
 
-    fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    if(method=='optim'){
+        fit <- optim(start, fn = measerr_mle_nll, lower=lower, method=optim_method, hessian=TRUE, control=list(maxit=maxit))
+    } else { # method='mle2'
+                
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
+
+        measerr_mle_nll_mle <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=maxit),method=optim_method)
+    }
 
     return(fit)
 }
+
+## Experimental, but probably works. 
+measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), coder_formulas=list(x.obs.0~x, x.obs.1~x), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
+
+    ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. 
+    # this time we never get to observe the true X
+    outcome.model.matrix <- model.matrix(outcome_formula, df)
+    proxy.model.matrix <- model.matrix(proxy_formula, df)
+    response.var <- all.vars(outcome_formula)[1]
+    proxy.var <- all.vars(proxy_formula)[1]
+    param.var <- all.vars(truth_formula)[1]
+    truth.var<- all.vars(truth_formula)[1]
+    y <- with(df,eval(parse(text=response.var)))
+
+    nll <- function(params){
+        param.idx <- 1
+        n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+        outcome.params <- params[param.idx:n.outcome.model.covars]
+        param.idx <- param.idx + n.outcome.model.covars
+
+
+        if(outcome_family$family == "gaussian"){
+            sigma.y <- params[param.idx]
+            param.idx <- param.idx + 1
+        }
+
+
+        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+        proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+        param.idx <- param.idx + n.proxy.model.covars
+        
+        df.temp <- copy(df)
+
+        if((truth_family$family == "binomial")
+           & (truth_family$link=='logit')){
+            integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+            ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+            for(i in 1:nrow(integrate.grid)){
+                # setup the dataframe for this row
+                row <- integrate.grid[i,]
+
+                df.temp[[param.var]] <- row[[1]]
+                ci <- 2
+                for(coder_formula in coder_formulas){
+                    coder.var <- all.vars(coder_formula)[1]
+                    df.temp[[coder.var]] <- row[[ci]]
+                    ci <- ci + 1 
+                }
+                
+                outcome.model.matrix <- model.matrix(outcome_formula, df.temp)                
+                if(outcome_family$family == "gaussian"){
+                    ll.y <- dnorm(y, outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=TRUE)
+                }
+
+                if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+                    proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+                    ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+                    proxyvar <- with(df.temp,eval(parse(text=proxy.var)))
+                    ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+                    ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+                }
+
+                ## probability of the coded variables
+                coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+                ci <- 1
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+                    param.idx <- param.idx + n.coder.model.covars
+                    coder.var <- all.vars(coder_formula)[1]
+                    x.obs <- with(df.temp, eval(parse(text=coder.var)))
+                    true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+                    ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+                    ll.coder[x.obs==1] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==1,]),log=TRUE)
+                    ll.coder[x.obs==0] <- plogis(coder.params %*% t(coder.model.matrix[x.obs==0,]),log=TRUE,lower.tail=FALSE)
+
+                    # don't count when we know the observed value, unless we're accounting for observed value
+                    ll.coder[(!is.na(true.codervar)) & (true.codervar != x.obs)] <- NA
+                    coder.lls[,ci] <- ll.coder
+                    ci <- ci + 1
+                }
+                
+                truth.model.matrix <- model.matrix(truth_formula, df.temp)
+                n.truth.model.covars <- dim(truth.model.matrix)[2]
+                truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
+
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    param.idx <- param.idx - n.coder.model.covars
+                }
+
+                x <- with(df.temp, eval(parse(text=truth.var)))
+                ll.truth <- vector(mode='numeric', length=dim(truth.model.matrix)[1])
+                ll.truth[x==1] <- plogis(truth.params %*% t(truth.model.matrix[x==1,]), log=TRUE)
+                ll.truth[x==0] <- plogis(truth.params %*% t(truth.model.matrix[x==0,]), log=TRUE, lower.tail=FALSE)
+
+                true.truthvar <- df[[all.vars(truth_formula)[1]]]
+                
+                if(!is.null(true.truthvar)){
+                                        # ll.truth[(!is.na(true.truthvar)) & (true.truthvar != truthvar)] <- -Inf
+                    # ll.truth[(!is.na(true.truthvar)) & (true.truthvar == truthvar)] <- 0
+                }
+                ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,sum) + ll.truth
+                
+            }
+
+            lls <- rowLogSumExps(ll.parts,na.rm=TRUE)
+
+        ## likelihood of observed data 
+            target <- -1 * sum(lls)
+            return(target)
+        }
+    }
+        
+    outcome.params <- colnames(model.matrix(outcome_formula,df))
+    lower <- rep(-Inf, length(outcome.params))
+
+    if(outcome_family$family=='gaussian'){
+        params <- c(outcome.params, 'sigma_y')
+        lower <- c(lower, 0.00001)
+    } else {
+        params <- outcome.params
+    }
+
+    proxy.params <- colnames(model.matrix(proxy_formula, df))
+    params <- c(params, paste0('proxy_',proxy.params))
+    positive.params <- paste0('proxy_',truth.var)
+    lower <- c(lower, rep(-Inf, length(proxy.params)))
+    names(lower) <- params
+    lower[positive.params] <- 0.01
+    ci <- 0
+    
+    for(coder_formula in coder_formulas){
+        coder.params <- colnames(model.matrix(coder_formula,df))
+        params <- c(params, paste0('coder_',ci,coder.params))
+        positive.params <- paste0('coder_', ci, truth.var)
+        ci <- ci + 1
+        lower <- c(lower, rep(-Inf, length(coder.params)))
+        names(lower) <- params
+        lower[positive.params] <- 0.01
+    }
+
+    truth.params <- colnames(model.matrix(truth_formula, df))
+    params <- c(params, paste0('truth_', truth.params))
+    lower <- c(lower, rep(-Inf, length(truth.params)))
+    start <- rep(0.1,length(params))
+    names(start) <- params
+    names(lower) <- params
+    
+    if(method=='optim'){
+        print(start)
+        fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
+    } else {
+                
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+        print(quoted.names)
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+        measerr_mle_nll <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, method='L-BFGS-B',control=list(maxit=1e6))
+    }
+
+    return(fit)
+}
+
+## Experimental, and does not work.
+measerr_irr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), coder_formulas=list(y.obs.0~y+w_pred+y.obs.1,y.obs.1~y+w_pred+y.obs.0), proxy_formula=w_pred~y, proxy_family=binomial(link='logit'),method='optim'){
+    integrate.grid <- expand.grid(replicate(1 + length(coder_formulas), c(0,1), simplify=FALSE))
+#    print(integrate.grid)
+
+
+    outcome.model.matrix <- model.matrix(outcome_formula, df)
+    n.outcome.model.covars <- dim(outcome.model.matrix)[2]
+
+
+    ### in this scenario, the ground truth also has measurement error, but we have repeated measures for it. 
+    # this time we never get to observe the true X
+    nll <- function(params){
+        param.idx <- 1
+        outcome.params <- params[param.idx:n.outcome.model.covars]
+        param.idx <- param.idx + n.outcome.model.covars
+        proxy.model.matrix <- model.matrix(proxy_formula, df)
+        n.proxy.model.covars <- dim(proxy.model.matrix)[2]
+        response.var <- all.vars(outcome_formula)[1]
+
+        if(outcome_family$family == "gaussian"){
+            sigma.y <- params[param.idx]
+            param.idx <- param.idx + 1
+        }
+
+        proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
+        param.idx <- param.idx + n.proxy.model.covars
+
+        df.temp <- copy(df)
+
+        if((outcome_family$family == "binomial")
+           & (outcome_family$link=='logit')){
+            ll.parts <- matrix(nrow=nrow(df),ncol=nrow(integrate.grid))
+            for(i in 1:nrow(integrate.grid)){
+                # setup the dataframe for this row
+                row <- integrate.grid[i,]
+
+                df.temp[[response.var]] <- row[[1]]
+                ci <- 2
+                for(coder_formula in coder_formulas){
+                    codervar <- all.vars(coder_formula)[1]
+                    df.temp[[codervar]] <- row[[ci]]
+                    ci <- ci + 1 
+                }
+                
+                outcome.model.matrix <- model.matrix(outcome_formula, df.temp)                
+                if(outcome_family$family == "gaussian"){
+                    ll.y <- dnorm(df.temp[[response.var]], outcome.params %*% t(outcome.model.matrix), sd=sigma.y, log=T)
+                }
+
+                if(outcome_family$family == "binomial" & (outcome_family$link=='logit')){
+                    ll.y <- vector(mode='numeric',length=nrow(df.temp))
+                    ll.y[df.temp[[response.var]]==1] <- plogis( outcome.params %*% t(outcome.model.matrix), log=TRUE)
+                    ll.y[df.temp[[response.var]]==0] <- plogis( outcome.params %*% t(outcome.model.matrix), log=TRUE,lower.tail=FALSE)
+                }
+
+                if(proxy_family$family=="binomial" & (proxy_family$link=='logit')){
+                    proxy.model.matrix <- model.matrix(proxy_formula, df.temp)
+                    ll.w <- vector(mode='numeric', length=dim(proxy.model.matrix)[1])
+                    proxyvar <- with(df.temp,eval(parse(text=all.vars(proxy_formula)[1])))
+                    ll.w[proxyvar==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==1,]),log=TRUE)
+                    ll.w[proxyvar==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxyvar==0,]),log=TRUE,lower.tail=FALSE)
+                }
+
+                ## probability of the coded variables
+                coder.lls <- matrix(nrow=nrow(df.temp),ncol=length(coder_formulas))
+                ci <- 1
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    coder.params <- params[param.idx:(n.coder.model.covars + param.idx - 1)]
+                    param.idx <- param.idx + n.coder.model.covars
+                    codervar <- with(df.temp, eval(parse(text=all.vars(coder_formula)[1])))
+                    true.codervar <- df[[all.vars(coder_formula)[1]]]
+
+                    ll.coder <- vector(mode='numeric', length=dim(coder.model.matrix)[1])
+                    ll.coder[codervar==1] <- plogis(coder.params %*% t(coder.model.matrix[codervar==1,]),log=TRUE)
+                    ll.coder[codervar==0] <- plogis(coder.params %*% t(coder.model.matrix[codervar==0,]),log=TRUE,lower.tail=FALSE)
+
+                    # don't count when we know the observed value, unless we're accounting for observed value
+                    ll.coder[(!is.na(true.codervar)) & (true.codervar != codervar)] <- NA
+                    coder.lls[,ci] <- ll.coder
+                    ci <- ci + 1
+                }
+
+                for(coder_formula in coder_formulas){
+                    coder.model.matrix <- model.matrix(coder_formula, df.temp)
+                    n.coder.model.covars <- dim(coder.model.matrix)[2]
+                    param.idx <- param.idx - n.coder.model.covars
+                }
+
+                ll.parts[,i] <- ll.y + ll.w + apply(coder.lls,1,function(x) sum(x)) 
+                
+            }
+
+            lls <- rowLogSumExps(ll.parts,na.rm=TRUE)
+
+            ## likelihood of observed data 
+            target <- -1 * sum(lls)
+#            print(target)
+#            print(params)
+            return(target)
+        }
+    }
+        
+    outcome.params <- colnames(model.matrix(outcome_formula,df))
+    response.var <- all.vars(outcome_formula)[1]
+    lower <- rep(-Inf, length(outcome.params))
+
+    if(outcome_family$family=='gaussian'){
+        params <- c(outcome.params, 'sigma_y')
+        lower <- c(lower, 0.00001)
+    } else {
+        params <- outcome.params
+    }
+
+    ## constrain the model of the coder and proxy vars
+    ## this is to ensure identifiability
+    ## it is a safe assumption because the coders aren't hostile (wrong more often than right)
+    ## so we can assume that y ~Bw, B is positive
+    proxy.params <- colnames(model.matrix(proxy_formula, df))
+    positive.params <- paste0('proxy_',response.var)
+    params <- c(params, paste0('proxy_',proxy.params))
+    lower <- c(lower, rep(-Inf, length(proxy.params)))
+    names(lower) <- params
+    lower[positive.params] <- 0.001
+
+    ci <- 0
+    for(coder_formula in coder_formulas){
+        coder.params <- colnames(model.matrix(coder_formula,df))
+        latent.coder.params <- coder.params %in% response.var
+        params <- c(params, paste0('coder_',ci,coder.params))
+        positive.params <- paste0('coder_',ci,response.var)
+        ci <- ci + 1
+        lower <- c(lower, rep(-Inf, length(coder.params)))
+        names(lower) <-params
+        lower[positive.params] <- 0.001
+    }
+
+    ## init by using the "loco model"
+    temp.df <- copy(df)
+    temp.df <- temp.df[y.obs.1 == y.obs.0, y:=y.obs.1]
+    loco.model <- glm(outcome_formula, temp.df, family=outcome_family)
+    
+    start <- rep(1,length(params))
+    names(start) <- params
+    start[names(coef(loco.model))] <- coef(loco.model)
+    names(lower) <- params
+    if(method=='optim'){
+        print(lower)
+        fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE,control=list(maxit=1e6))
+    } else {
+                
+        quoted.names <- gsub("[\\(\\)]",'',names(start))
+        print(quoted.names)
+        text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
+
+        measerr_mle_nll <- eval(parse(text=text))
+        names(start) <- quoted.names
+        names(lower) <- quoted.names
+        fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
+    }
+
+    return(fit)
+}
+

simulations/pl_methods.R

@@ -0,0 +1,86 @@
+library(stats4)
+library(bbmle)
+library(matrixStats)
+
+zhang.mle.dv <- function(df){
+    df.obs <- df[!is.na(y.obs)]
+    df.unobs <- df[is.na(y.obs)]
+
+    fp <- df.obs[(w_pred==1) & (y.obs != w_pred),.N]
+    tn <- df.obs[(w_pred == 0) & (y.obs == w_pred),.N]
+    fpr <- fp / (fp+tn)
+
+    fn <- df.obs[(w_pred==0) & (y.obs != w_pred), .N]
+    tp <- df.obs[(w_pred==1) & (y.obs == w_pred),.N]
+    fnr <- fn / (fn+tp)
+
+    nll <- function(B0=0, Bxy=0, Bzy=0){
+
+
+        ## observed case
+        ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
+        ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
+        ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
+
+        ll <- sum(ll.y.obs)
+
+        pi.y.1 <- with(df.unobs,plogis(B0 + Bxy * x + Bzy*z, log=T))
+        #pi.y.0 <- with(df.unobs,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
+
+        lls <- with(df.unobs, colLogSumExps(rbind(w_pred * colLogSumExps(rbind(log(fpr), log(1 - fnr - fpr)+pi.y.1)),
+        (1-w_pred) * (log(1-fpr) - exp(log(1-fnr-fpr)+pi.y.1)))))
+    
+        ll <- ll + sum(lls)
+#        print(paste0(B0,Bxy,Bzy))
+#        print(ll)
+        return(-ll)
+    }    
+    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=c(B0=-Inf, Bxy=-Inf, Bzy=-Inf),
+                   upper=c(B0=Inf, Bxy=Inf, Bzy=Inf))
+    return(mlefit)
+}
+
+
+## model from Zhang's arxiv paper, with predictions for y
+## Zhang got this model from Hausman 1998
+zhang.mle.iv <- function(df){
+    df.obs <- df[!is.na(x.obs)]
+    df.unobs <- df[is.na(x.obs)]
+
+    tn <- df.obs[(w_pred == 0) & (x.obs == w_pred),.N]
+    fn <- df.obs[(w_pred==0) & (x.obs==1), .N]
+    npv <- tn / (tn + fn)
+
+
+    tp <- df.obs[(w_pred==1) & (x.obs == w_pred),.N]
+    fp <- df.obs[(w_pred==1) & (x.obs == 0),.N]
+    ppv <- tp / (tp + fp)
+
+    nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=9){
+
+    ## fpr = 1 - TNR
+    ### Problem: accounting for uncertainty in ppv / npv
+
+    ## fnr = 1 - TPR
+    ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
+
+    ll <- sum(ll.y.obs)
+    # unobserved case; integrate out x
+    ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
+    ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
+
+    ## case x == 1
+    lls.x.1 <- colLogSumExps(rbind(log(ppv) + ll.x.1, log(1-ppv) + ll.x.0))
+    
+    ## case x == 0
+    lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
+
+    lls <- colLogSumExps(rbind(df.unobs$w_pred * lls.x.1, (1-df.unobs$w_pred) * lls.x.0))
+        
+    ll <- ll + sum(lls)
+
+    }    
+    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.00001, B0=-Inf, Bxy=-Inf, Bzy=-Inf),
+                   upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf),method='L-BFGS-B')
+    return(mlefit)
+}

simulations/plot_dv_example.R

@@ -6,53 +6,52 @@ library(filelock)
 library(argparser)
 
 parser <- arg_parser("Simulate data and fit corrected models.")
-parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--infile", default="robustness_3_dv.feather", help="name of the file to read.")
+parser <- add_argument(parser, "--remember-file", default="remembr.RDS", help="name of the remember file.")
 parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
 args <- parse_args(parser)
 
+## summarize.estimator <- function(df, suffix='naive', coefname='x'){
 
+##     part <- df[,c('N',
+##                   'm',
+##                   'Bxy',
+##                   paste0('B',coefname,'y.est.',suffix),
+##                   paste0('B',coefname,'y.ci.lower.',suffix),
+##                   paste0('B',coefname,'y.ci.upper.',suffix),
+##                   'y_explained_variance',
+##                   'Bzy'
+##                   ),
+##                with=FALSE]
     
-summarize.estimator <- function(df, suffix='naive', coefname='x'){
+##     true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+##     zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+##     bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+##     sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
 
-    part <- df[,c('N',
+##     part <- part[,':='(true.in.ci = true.in.ci,
-                  'm',
+##                        zero.in.ci = zero.in.ci,
-                  'Bxy',
+##                        bias=bias,
-                  paste0('B',coefname,'y.est.',suffix),
+##                        sign.correct =sign.correct)]
-                  paste0('B',coefname,'y.ci.lower.',suffix),
-                  paste0('B',coefname,'y.ci.upper.',suffix),
-                  'y_explained_variance',
-                  'Bzy',
-                  'accuracy_imbalance_difference'
-                  ),
-               with=FALSE]
 
-    true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+##     part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
-    zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+##                           mean.bias = mean(bias),
-    bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
+##                           mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
-    sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+##                           var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+##                           est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
+##                           est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
+##                           N.sims = .N,
+##                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+##                           variable=coefname,
+##                           method=suffix
+##                           ),
+##                       by=c("N","m",'Bzy','y_explained_variance')
+##                       ]
     
-    part <- part[,':='(true.in.ci = true.in.ci,
+##     return(part.plot)
-                       zero.in.ci = zero.in.ci,
+## }
-                       bias=bias,
-                       sign.correct =sign.correct)]
-
-    part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
-                          mean.bias = mean(bias),
-                          mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
-                          var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
-                          est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
-                          est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
-                          N.sims = .N,
-                          p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                          variable=coefname,
-                          method=suffix
-                          ),
-                      by=c("N","m",'Bzy','accuracy_imbalance_difference','y_explained_variance')
-                      ]
-    
-    return(part.plot)
-}
 
+source("summarize_estimator.R")
 
 build_plot_dataset <- function(df){
 
@@ -85,12 +84,22 @@ build_plot_dataset <- function(df){
     return(plot.df)
 }
 
-
+change.remember.file(args$remember_file, clear=TRUE)
-df <- read_feather(args$infile)
+sims.df <- read_feather(args$infile)
-plot.df <- build_plot_dataset(df)
+plot.df <- build_plot_dataset(sims.df)
 
 remember(plot.df,args$name)
 
+set.remember.prefix(gsub("plot.df.","",args$name))
+
+remember(median(sims.df$cor.xz),'med.cor.xz')
+remember(median(sims.df$accuracy),'med.accuracy')
+remember(median(sims.df$error.cor.x),'med.error.cor.x')
+remember(median(sims.df$error.cor.z),'med.error.cor.z')
+remember(median(sims.df$lik.ratio),'med.lik.ratio')
+
+
+
 
 ## df[gmm.ER_pval<0.05]
 ## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),

simulations/plot_example.R

@@ -5,52 +5,58 @@ library(ggplot2)
 library(filelock)
 library(argparser)
 
+source("summarize_estimator.R")
+
+
 parser <- arg_parser("Simulate data and fit corrected models.")
-parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--infile", default="robustness_2.feather", help="name of the file to read.")
+parser <- add_argument(parser, "--remember-file", default="remembr.RDS", help="name of the remember file.")
 parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
 args <- parse_args(parser)
 
-summarize.estimator <- function(df, suffix='naive', coefname='x'){
 
-    part <- df[,c('N',
-                  'm',
-                  'Bxy',
-                  paste0('B',coefname,'y.est.',suffix),
-                  paste0('B',coefname,'y.ci.lower.',suffix),
-                  paste0('B',coefname,'y.ci.upper.',suffix),
-                  'y_explained_variance',
-                  'Bzx',
-                  'Bzy',
-                  'accuracy_imbalance_difference'
-                  ),
-               with=FALSE]
 
-    true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+## summarize.estimator <- function(df, suffix='naive', coefname='x'){
-    zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
-    bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
-    sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
 
-    part <- part[,':='(true.in.ci = true.in.ci,
+##     part <- df[,c('N',
-                       zero.in.ci = zero.in.ci,
+##                   'm',
-                       bias=bias,
+##                   'Bxy',
-                       sign.correct =sign.correct)]
+##                   paste0('B',coefname,'y.est.',suffix),
+##                   paste0('B',coefname,'y.ci.lower.',suffix),
+##                   paste0('B',coefname,'y.ci.upper.',suffix),
+##                   'y_explained_variance',
+##                   'Bzx',
+##                   'Bzy',
+##                   'accuracy_imbalance_difference'
+##                   ),
+##                with=FALSE]
     
-    part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+##     true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
-                          mean.bias = mean(bias),
+##     zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
-                          mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+##     bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
-                          var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+##     sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
-                          est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
-                          est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
-                          N.sims = .N,
-                          p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
-                          variable=coefname,
-                          method=suffix
-                          ),
-                      by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
-                      ]
 
-    return(part.plot)
+##     part <- part[,':='(true.in.ci = true.in.ci,
-}
+##                        zero.in.ci = zero.in.ci,
+##                        bias=bias,
+##                        sign.correct =sign.correct)]
+
+##     part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+##                           mean.bias = mean(bias),
+##                           mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+##                           var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
+##                           est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
+##                           est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
+##                           N.sims = .
+##                           p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+##                           variable=coefname,
+##                           method=suffix
+##                           ),
+##                       by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
+##                       ]
+    
+##     return(part.plot)
+## }
 
 build_plot_dataset <- function(df){
     
@@ -70,13 +76,13 @@ build_plot_dataset <- function(df){
 
     z.amelia.full <- summarize.estimator(df, 'amelia.full', 'z')
     
-    x.mecor <- summarize.estimator(df, 'mecor', 'x')
+    ## x.mecor <- summarize.estimator(df, 'mecor', 'x')
 
-    z.mecor <- summarize.estimator(df, 'mecor', 'z')
+    ## z.mecor <- summarize.estimator(df, 'mecor', 'z')
 
-    x.mecor <- summarize.estimator(df, 'mecor', 'x')
+    ## x.mecor <- summarize.estimator(df, 'mecor', 'x')
 
-    z.mecor <- summarize.estimator(df, 'mecor', 'z')
+    ## z.mecor <- summarize.estimator(df, 'mecor', 'z')
 
     x.mle <- summarize.estimator(df, 'mle', 'x')
 
@@ -91,30 +97,48 @@ build_plot_dataset <- function(df){
     z.gmm <- summarize.estimator(df, 'gmm', 'z')
 
     accuracy <- df[,mean(accuracy)]
-    plot.df <- rbindlist(list(x.true,z.true,x.naive,z.naive,x.amelia.full,z.amelia.full,x.mecor, z.mecor, x.gmm, z.gmm, x.feasible, z.feasible,z.mle, x.mle, x.zhang, z.zhang, x.gmm, z.gmm),use.names=T)
+    plot.df <- rbindlist(list(x.true,z.true,x.naive,z.naive,x.amelia.full,z.amelia.full,x.gmm, z.gmm, x.feasible, z.feasible,z.mle, x.mle, x.zhang, z.zhang, x.gmm, z.gmm),use.names=T)
     plot.df[,accuracy := accuracy]
     plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
     return(plot.df)
 }
 
 
-plot.df <- read_feather(args$infile)
+sims.df <- read_feather(args$infile)
+unique(sims.df[,.N,by=.(N,m)])
+print(unique(sims.df$N))
 
 # df <- df[apply(df,1,function(x) !any(is.na(x)))]
 
-if(!('Bzx' %in% names(plot.df)))
+if(!('Bzx' %in% names(sims.df)))
-    plot.df[,Bzx:=NA]
+    sims.df[,Bzx:=NA]
 
-if(!('accuracy_imbalance_difference' %in% names(plot.df)))
+if(!('accuracy_imbalance_difference' %in% names(sims.df)))
-    plot.df[,accuracy_imbalance_difference:=NA]
+    sims.df[,accuracy_imbalance_difference:=NA]
 
-unique(plot.df[,'accuracy_imbalance_difference'])
+unique(sims.df[,'accuracy_imbalance_difference'])
 
+change.remember.file(args$remember_file, clear=TRUE)
 #plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
-plot.df <- build_plot_dataset(plot.df)
+plot.df <- build_plot_dataset(sims.df)
 
 remember(plot.df,args$name)
 
+set.remember.prefix(gsub("plot.df.","",args$name))
+
+remember(median(sims.df$cor.xz),'med.cor.xz')
+remember(median(sims.df$accuracy),'med.accuracy')
+remember(median(sims.df$accuracy.y0),'med.accuracy.y0')
+remember(median(sims.df$accuracy.y1),'med.accuracy.y1')
+remember(median(sims.df$fpr),'med.fpr')
+remember(median(sims.df$fpr.y0),'med.fpr.y0')
+remember(median(sims.df$fpr.y1),'med.fpr.y1')
+remember(median(sims.df$fnr),'med.fnr')
+remember(median(sims.df$fnr.y0),'med.fnr.y0')
+remember(median(sims.df$fnr.y1),'med.fnr.y1')
+
+remember(median(sims.df$cor.resid.w_pred),'cor.resid.w_pred')
+
 #ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
 
 ## ## ## df[gmm.ER_pval<0.05]

simulations/plot_irr_dv_example.R

@@ -0,0 +1,73 @@
+source("RemembR/R/RemembeR.R")
+library(arrow)
+library(data.table)
+library(ggplot2)
+library(filelock)
+library(argparser)
+
+parser <- arg_parser("Simulate data and fit corrected models.")
+parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
+args <- parse_args(parser)
+source("summarize_estimator.R")
+
+build_plot_dataset <- function(df){
+    
+    x.true <-  summarize.estimator(df, 'true','x')
+
+    z.true <-  summarize.estimator(df, 'true','z')
+
+    x.naive <-  summarize.estimator(df, 'naive','x')
+
+    z.naive <-  summarize.estimator(df, 'naive','z')
+
+    x.loa0.feasible <- summarize.estimator(df, 'loa0.feasible','x')
+    
+    z.loa0.feasible <- summarize.estimator(df,'loa0.feasible','z')
+
+    x.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'x')
+
+    z.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'z')
+
+    x.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'x')
+
+    z.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'z')
+
+    x.loco.mle <- summarize.estimator(df, 'loco.mle', 'x')
+
+    z.loco.mle <- summarize.estimator(df, 'loco.mle', 'z')
+
+
+    z.loco.amelia <- summarize.estimator(df, 'amelia.full', 'z')
+    x.loco.amelia <- summarize.estimator(df, 'amelia.full', 'x')
+
+    z.loco.zhang <- summarize.estimator(df, 'zhang', 'z')
+    x.loco.zhang <- summarize.estimator(df, 'zhang', 'x')
+
+    accuracy <- df[,mean(accuracy)]
+    plot.df <- rbindlist(list(x.true,z.true,x.loa0.feasible,z.loa0.feasible,x.naive,z.naive,x.loa0.mle,z.loa0.mle,x.loco.feasible, z.loco.feasible, z.loco.mle, x.loco.mle, x.loco.amelia, z.loco.amelia, z.loco.zhang, x.loco.zhang),use.names=T)
+    plot.df[,accuracy := accuracy]
+    plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
+    return(plot.df)
+}
+
+
+plot.df <- read_feather(args$infile)
+print(unique(plot.df$N))
+
+# df <- df[apply(df,1,function(x) !any(is.na(x)))]
+
+if(!('Bzx' %in% names(plot.df)))
+    plot.df[,Bzx:=NA]
+
+if(!('accuracy_imbalance_difference' %in% names(plot.df)))
+    plot.df[,accuracy_imbalance_difference:=NA]
+
+unique(plot.df[,'accuracy_imbalance_difference'])
+
+#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
+plot.df <- build_plot_dataset(plot.df)
+
+change.remember.file("remember_irr.RDS",clear=TRUE)
+
+remember(plot.df,args$name)

simulations/plot_irr_example.R

@@ -0,0 +1,152 @@
+source("RemembR/R/RemembeR.R")
+library(arrow)
+library(data.table)
+library(ggplot2)
+library(filelock)
+library(argparser)
+
+parser <- arg_parser("Simulate data and fit corrected models.")
+parser <- add_argument(parser, "--infile", default="", help="name of the file to read.")
+parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
+args <- parse_args(parser)
+source("summarize_estimator.R")
+
+build_plot_dataset <- function(df){
+    
+    x.true <-  summarize.estimator(df, 'true','x')
+
+    z.true <-  summarize.estimator(df, 'true','z')
+
+    x.naive <-  summarize.estimator(df, 'naive','x')
+
+    z.naive <-  summarize.estimator(df, 'naive','z')
+
+    x.loa0.feasible <- summarize.estimator(df, 'loa0.feasible','x')
+    
+    z.loa0.feasible <- summarize.estimator(df,'loa0.feasible','z')
+
+    x.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'x')
+
+    z.loa0.mle <- summarize.estimator(df, 'loa0.mle', 'z')
+
+    x.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'x')
+
+    z.loco.feasible <- summarize.estimator(df, 'loco.feasible', 'z')
+
+    x.loco.mle <- summarize.estimator(df, 'loco.mle', 'x')
+
+    z.loco.mle <- summarize.estimator(df, 'loco.mle', 'z')
+
+    x.loco.mle <- summarize.estimator(df, 'loco.mle', 'x')
+
+    z.loco.amelia <- summarize.estimator(df, 'amelia.full', 'z')
+    x.loco.amelia <- summarize.estimator(df, 'amelia.full', 'x')
+
+    z.loco.zhang <- summarize.estimator(df, 'zhang', 'z')
+    x.loco.zhang <- summarize.estimator(df, 'zhang', 'x')
+
+
+    z.loco.gmm <- summarize.estimator(df, 'gmm', 'z')
+    x.loco.gmm <- summarize.estimator(df, 'gmm', 'x')
+
+    
+
+
+    ## x.mle <- summarize.estimator(df, 'mle', 'x')
+
+    ## z.mle <- summarize.estimator(df, 'mle', 'z')
+
+    accuracy <- df[,mean(accuracy)]
+    plot.df <- rbindlist(list(x.true,z.true,x.loa0.feasible,z.loa0.feasible,x.loa0.mle,z.loa0.mle,x.loco.feasible, z.loco.feasible, x.loco.mle, z.loco.mle, x.loco.amelia, z.loco.amelia,x.loco.zhang, z.loco.zhang,x.loco.gmm, z.loco.gmm,x.naive,z.naive),use.names=T)
+    plot.df[,accuracy := accuracy]
+    plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
+    return(plot.df)
+}
+
+
+sims.df <- read_feather(args$infile)
+print(unique(sims.df$N))
+
+# df <- df[apply(df,1,function(x) !any(is.na(x)))]
+
+if(!('Bzx' %in% names(sims.df)))
+    sims.df[,Bzx:=NA]
+
+if(!('accuracy_imbalance_difference' %in% names(sims.df)))
+    sims.df[,accuracy_imbalance_difference:=NA]
+
+unique(sims.df[,'accuracy_imbalance_difference'])
+
+#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
+plot.df <- build_plot_dataset(sims.df)
+change.remember.file("remember_irr.RDS",clear=TRUE)
+remember(plot.df,args$name)
+
+
+set.remember.prefix(gsub("plot.df.","",args$name))
+remember(median(sims.df$loco.accuracy),'med.loco.acc')
+
+#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
+
+## ## ## df[gmm.ER_pval<0.05]
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='x') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(N~m,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='z') & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=-0.1),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+
+## x.mle <- df[,.(N,m,Bxy.est.mle,Bxy.ci.lower.mle, Bxy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
+## x.mle.plot <- x.mle[,.(mean.est = mean(Bxy.est.mle),
+##                        var.est = var(Bxy.est.mle),
+##                        N.sims = .N,
+##                        variable='z',
+##                        method='Bespoke MLE'
+##                        ),
+##                     by=c("N","m",'y_explained_variance', 'Bzx', 'Bzy','accuracy_imbalance_difference')]
+
+## z.mle <- df[,.(N,m,Bzy.est.mle,Bzy.ci.lower.mle, Bzy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
+
+## z.mle.plot <- z.mle[,.(mean.est = mean(Bzy.est.mle),
+##                        var.est = var(Bzy.est.mle),
+##                        N.sims = .N,
+##                        variable='z',
+##                        method='Bespoke MLE'
+##                        ),
+##                     by=c("N","m",'y_explained_variance','Bzx')]
+
+## plot.df <- z.mle.plot
+## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
+##                                    N=factor(N),
+##                                    m=factor(m))]
+
+## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & (method!="Multiple imputation (Classifier features unobserved)")]
+## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
+## p <- p + geom_hline(aes(yintercept=0.2),linetype=2)
+
+## p <- p + geom_pointrange() + facet_grid(m~Bzx, Bzy,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
+## print(p)
+
+
+## ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
+
+## ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 
+
+## ## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size") 

simulations/robustness_check_notes.md

@@ -0,0 +1,39 @@
+# robustness\_1.RDS
+
+Tests how robust the MLE method for independent variables with differential error is when the model for $X$ is less precise. In the main paper, we include $Z$ on the right-hand-side of the `truth_formula`. 
+In this robustness check, the `truth_formula` is an intercept-only model.
+The stats are in the list named `robustness_1` in the `.RDS` 
+# robustness\_1\_dv.RDS
+
+Like `robustness\_1.RDS` but with a less precise model for $w_pred$.  In the main paper, we included $Z$ in the `proxy_formula`. In this robustness check, we do not.
+
+# robustness_2.RDS
+
+This is just example 1 with varying levels of classifier accuracy indicated by the `prediction_accuracy` variable.. 
+
+# robustness_2_dv.RDS
+
+Example 3 with varying levels of classifier accuracy indicated by the `prediction_accuracy` variable.
+
+# robustness_3.RDS
+
+Example 1 with varying levels of skewness in the classified variable. The variable `Px` is the baserate of $X$ and controls the skewness of $X$.
+It probably makes more sense to report the mean of $X$ instead of `Px` in the supplement.
+
+# robustness_3_dv.RDS
+
+Example 3 with varying levels of skewness in the classified variable. The variable `B0` is the intercept of the main model and controls the skewness of $Y$.
+It probably makes more sense to report the mean of $Y$ instead of B0 in the supplement. 
+
+# robustness_4.RDS
+
+Example 2 with varying amounts of differential error. The variable `y_bias` controls the amount of differential error.
+It probably makes more sense to report the corrleation between $Y$ and $X-~$, or the difference in accuracy from when when $Y=1$ to $Y=0$ in the supplement instead of `y_bias`.
+
+# robustness_4_dv.RDS
+
+Example 4 with varying amounts of bias. The variable `z_bias` controls the amount of differential error.
+It probably makes more sense to report the corrleation between $Z$ and $Y-W$, or the difference in accuracy from when when $Z=1$ to $Z=0$ in the supplement instead of `z_bias`.
+
+
+

simulations/run_job.sbatch

@@ -0,0 +1,18 @@
+#!/bin/bash
+#SBATCH --job-name="simulate measurement error models"
+## Allocation Definition
+#SBATCH --account=comdata-ckpt
+#SBATCH --partition=ckpt
+## Resources
+#SBATCH --nodes=1    
+## Walltime (4 hours)
+#SBATCH --time=4:00:00
+## Memory per node
+#SBATCH --mem=4G
+#SBATCH --cpus-per-task=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
+#SBATCH --output=simulation_jobs/%A_%a.out
+#SBATCH --error=simulation_jobs/%A_%a.err
+echo "$@"
+"$@"

simulations/run_simulation.sbatch

@@ -1,21 +1,23 @@
 #!/bin/bash
 #SBATCH --job-name="simulate measurement error models"
 ## Allocation Definition
-#SBATCH --account=comdata
+#SBATCH --account=comdata-ckpt
-#SBATCH --partition=compute-bigmem
+#SBATCH --partition=ckpt
 ## Resources
 #SBATCH --nodes=1    
-## Walltime (12 hours)
+## Walltime (4 hours)
-#SBATCH --time=1:00:00
+#SBATCH --time=4:00:00
 ## Memory per node
-#SBATCH --mem=8G
+#SBATCH --mem=4G
 #SBATCH --cpus-per-task=1
 #SBATCH --ntasks-per-node=1
 #SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
 #SBATCH --output=simulation_jobs/%A_%a.out
 #SBATCH --error=simulation_jobs/%A_%a.err
+source ~/.bashrc
 
 TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1))
 TASK_CALL=$(sed -n ${TASK_NUM}p $2)
+echo ${TASK_CALL}
 ${TASK_CALL}
 

simulations/simulation_base.R

@@ -7,6 +7,7 @@ library(Zelig)
 library(bbmle)
 library(matrixStats) # for numerically stable logsumexps
 
+source("pl_methods.R")
 source("measerr_methods.R") ## for my more generic function.
 
 ## This uses the pseudolikelihood approach from Carroll page 349.
@@ -37,84 +38,6 @@ my.pseudo.mle <- function(df){
 }
 
  
-## model from Zhang's arxiv paper, with predictions for y
-## Zhang got this model from Hausman 1998
-### I think this is actually eqivalent to the pseudo.mle method
-zhang.mle.iv <- function(df){
-    nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1, ppv=0.9, npv=0.9){
-    df.obs <- df[!is.na(x.obs)]
-    df.unobs <- df[is.na(x.obs)]
-
-    ## fpr = 1 - TNR
-    ### Problem: accounting for uncertainty in ppv / npv
-    
-    ll.w1x1.obs <- with(df.obs[(w_pred==1)], dbinom(x.obs,size=1,prob=ppv,log=T))
-    ll.w0x0.obs <- with(df.obs[(w_pred==0)], dbinom(1-x.obs,size=1,prob=npv,log=T))
-
-    ## fnr = 1 - TPR
-    ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
-    ll <- sum(ll.y.obs)
-    ll <- ll + sum(ll.w1x1.obs) + sum(ll.w0x0.obs)
-
-    # unobserved case; integrate out x
-    ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
-    ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
-
-    ## case x == 1
-    lls.x.1 <- colLogSumExps(rbind(log(ppv) + ll.x.1, log(1-ppv) + ll.x.0))
-    
-    ## case x == 0
-    lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
-
-    lls <- colLogSumExps(rbind(lls.x.1, lls.x.0))
-    ll <- ll + sum(lls)
-    return(-ll)
-    }    
-    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf,ppv=0.00001, npv=0.00001),
-                   upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf, ppv=0.99999,npv=0.99999),method='L-BFGS-B')
-    return(mlefit)
-}
-
-## this is equivalent to the pseudo-liklihood model from Carolla
-zhang.mle.dv <- function(df){
-
-    nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){
-    df.obs <- df[!is.na(y.obs)]
-
-    ## fpr = 1 - TNR
-    ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE))
-    ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE))
-
-    # observed case
-    ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
-    ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
-    ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
-
-    ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1)
-
-    # unobserved case; integrate out y
-    ## case y = 1
-    ll.y.1 <- vector(mode='numeric', length=nrow(df))
-    pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
-    ## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1)
-    lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1))
-    
-    ## case y = 0
-    ll.y.0 <- vector(mode='numeric', length=nrow(df))
-    pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
-
-    ## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0)
-    lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0))
-
-    lls <- colLogSumExps(rbind(lls.y.1, lls.y.0))
-    ll <- ll + sum(lls)
-    return(-ll)
-    }    
-    mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001),
-                   upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999))
-    return(mlefit)
-}
-
 ## This uses the likelihood approach from Carroll page 353.
 ## assumes that we have a good measurement error model
 my.mle <- function(df){
@@ -166,15 +89,27 @@ my.mle <- function(df){
     return(mlefit)
 }
 
-run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~y){
+run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~y, confint_method='quad'){
 
-    accuracy <- df[,mean(w_pred==y)]
+    (accuracy <- df[,mean(w_pred==y)])
     result <- append(result, list(accuracy=accuracy))
+    (error.cor.z <- cor(df$z, df$y - df$w_pred))
+    (error.cor.x <- cor(df$x, df$y - df$w_pred))
+    (error.cor.y <- cor(df$y, df$y - df$w_pred))
+    result <- append(result, list(error.cor.x = error.cor.x,
+                                  error.cor.z = error.cor.z,
+                                  error.cor.y = error.cor.y))
 
+    model.null <- glm(y~1, data=df,family=binomial(link='logit'))
     (model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit')))
+    (lik.ratio <- exp(logLik(model.true) - logLik(model.null)))
+
     true.ci.Bxy <- confint(model.true)['x',]
     true.ci.Bzy <- confint(model.true)['z',]
 
+    result <- append(result, list(cor.xz=cor(df$x,df$z)))
+    result <- append(result, list(lik.ratio=lik.ratio))
+
     result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
                                   Bzy.est.true=coef(model.true)['z'],
                                   Bxy.ci.upper.true = true.ci.Bxy[2],
@@ -211,15 +146,17 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
     naivecont.ci.Bxy <- confint(model.naive.cont)['x',]
     naivecont.ci.Bzy <- confint(model.naive.cont)['z',]
 
-    ## my implementatoin of liklihood based correction
+    ## my implementation of liklihood based correction
 
     temp.df <- copy(df)
     temp.df[,y:=y.obs]
+
     mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
-    fisher.info <- solve(mod.caroll.lik$hessian)
+    fischer.info <- solve(mod.caroll.lik$hessian)
     coef <- mod.caroll.lik$par
-    ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+    ci.upper <- coef + sqrt(diag(fischer.info)) * 1.96
-    ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+    ci.lower <- coef - sqrt(diag(fischer.info)) * 1.96
+
     result <- append(result,
                      list(Bxy.est.mle = coef['x'],
                           Bxy.ci.upper.mle = ci.upper['x'],
@@ -228,6 +165,19 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
                           Bzy.ci.upper.mle = ci.upper['z'],
                           Bzy.ci.lower.mle = ci.lower['z']))
 
+    mod.caroll.profile.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, method='bbmle')
+    coef <- coef(mod.caroll.profile.lik)
+    ci <- confint(mod.caroll.profile.lik, method='spline')
+    ci.lower <- ci[,'2.5 %']
+    ci.upper <- ci[,'97.5 %']
+
+    result <- append(result,
+                     list(Bxy.est.mle.profile = coef['x'],
+                          Bxy.ci.upper.mle.profile = ci.upper['x'],
+                          Bxy.ci.lower.mle.profile = ci.lower['x'],
+                          Bzy.est.mle.profile = coef['z'],
+                          Bzy.ci.upper.mle.profile = ci.upper['z'],
+                          Bzy.ci.lower.mle.profile = ci.lower['z']))
 
     ## my implementatoin of liklihood based correction
     mod.zhang <- zhang.mle.dv(df)
@@ -243,34 +193,37 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
                           Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
 
     
+
     # amelia says use normal distribution for binary variables.
+    amelia_result <- list(Bxy.est.amelia.full = NA,
+                          Bxy.ci.upper.amelia.full = NA,
+                          Bxy.ci.lower.amelia.full = NA,
+                          Bzy.est.amelia.full = NA,
+                          Bzy.ci.upper.amelia.full = NA,
+                          Bzy.ci.lower.amelia.full = NA
+                          )
+
     tryCatch({
-        amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'))
+        amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'),ords="y.obs")
-        mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+        mod.amelia.k <- zelig(y.obs~x+z, model='logit', data=amelia.out.k$imputations, cite=FALSE)
         (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
         est.x.mi <- coefse['x','Estimate']
         est.x.se <- coefse['x','Std.Error']
-        result <- append(result,
-                         list(Bxy.est.amelia.full = est.x.mi,
-                              Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
-                              Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
-                              ))
 
         est.z.mi <- coefse['z','Estimate']
         est.z.se <- coefse['z','Std.Error']
-
+        amelia_result <- list(Bxy.est.amelia.full = est.x.mi,
-        result <- append(result,
+                          Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
-                         list(Bzy.est.amelia.full = est.z.mi,
+                          Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se,
-                              Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+                          Bzy.est.amelia.full = est.z.mi,
-                              Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
+                          Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
-                              ))
+                          Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
-
+                          )
     },
     error = function(e){
-        message("An error occurred:\n",e)
+    result[['error']] <- e}
-        result$error <- paste0(result$error,'\n', e)
+    )
-    })
+    result <- append(result,amelia_result)
-
 
     return(result)
 
@@ -278,11 +231,36 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
 
 
 ## outcome_formula, proxy_formula, and truth_formula are passed to measerr_mle 
-run_simulation <-  function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x, truth_formula=x~z){
+run_simulation <-  function(df, result, outcome_formula=y~x+z, proxy_formula=NULL, truth_formula=NULL, confint_method='quad'){
 
     accuracy <- df[,mean(w_pred==x)]
-    result <- append(result, list(accuracy=accuracy))
+    accuracy.y0 <- df[y<=0,mean(w_pred==x)]
+    accuracy.y1 <- df[y>=0,mean(w_pred==x)]
+    cor.y.xi <- cor(df$x - df$w_pred, df$y)
 
+    fnr <- df[w_pred==0,mean(w_pred!=x)]
+    fnr.y0 <- df[(w_pred==0) & (y<=0),mean(w_pred!=x)]
+    fnr.y1 <- df[(w_pred==0) & (y>=0),mean(w_pred!=x)]
+
+    fpr <- df[w_pred==1,mean(w_pred!=x)]
+    fpr.y0 <- df[(w_pred==1) & (y<=0),mean(w_pred!=x)]
+    fpr.y1 <- df[(w_pred==1) & (y>=0),mean(w_pred!=x)]
+    cor.resid.w_pred <- cor(resid(lm(y~x+z,df)),df$w_pred)
+
+    result <- append(result, list(accuracy=accuracy,
+                                  accuracy.y0=accuracy.y0,
+                                  accuracy.y1=accuracy.y1,
+                                  cor.y.xi=cor.y.xi,
+                                  fnr=fnr,
+                                  fnr.y0=fnr.y0,
+                                  fnr.y1=fnr.y1,
+                                  fpr=fpr,
+                                  fpr.y0=fpr.y0,
+                                  fpr.y1=fpr.y1,
+                                  cor.resid.w_pred=cor.resid.w_pred
+                                  ))
+
+    result <- append(result, list(cor.xz=cor(df$x,df$z)))
     (model.true <- lm(y ~ x + z, data=df))
     true.ci.Bxy <- confint(model.true)['x',]
     true.ci.Bzy <- confint(model.true)['z',]
@@ -318,79 +296,120 @@ run_simulation <-  function(df, result, outcome_formula=y~x+z, proxy_formula=w_p
                                   Bzy.ci.upper.naive = naive.ci.Bzy[2],
                                   Bzy.ci.lower.naive = naive.ci.Bzy[1]))
 
+    amelia_result <- list(
+        Bxy.est.amelia.full = NULL,
+        Bxy.ci.upper.amelia.full = NULL,
+        Bxy.ci.lower.amelia.full = NULL,
+        Bzy.est.amelia.full = NULL,
+        Bzy.ci.upper.amelia.full = NULL,
+        Bzy.ci.lower.amelia.full = NULL
+        )
 
     tryCatch({
-    amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred'))
+        amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w'))
-    mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
+        mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
-    (coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
+        (coefse <- combine_coef_se(mod.amelia.k))
 
-    est.x.mi <- coefse['x.obs','Estimate']
+        est.x.mi <- coefse['x.obs','Estimate']
-    est.x.se <- coefse['x.obs','Std.Error']
+        est.x.se <- coefse['x.obs','Std.Error']
-    result <- append(result,
+        est.z.mi <- coefse['z','Estimate']
-                     list(Bxy.est.amelia.full = est.x.mi,
+        est.z.se <- coefse['z','Std.Error']
-                          Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
-                          Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
-                          ))
 
-    est.z.mi <- coefse['z','Estimate']
+        amelia_result <- list(Bxy.est.amelia.full = est.x.mi,
-    est.z.se <- coefse['z','Std.Error']
+                              Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
-
+                              Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se,
-    result <- append(result,
+                              Bzy.est.amelia.full = est.z.mi,
-                     list(Bzy.est.amelia.full = est.z.mi,
+                              Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
-                          Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
+                              Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
-                          Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
+                              )
-                          ))
 
     },
+
     error = function(e){
-        message("An error occurred:\n",e)
+        result[['error']] <- e}
-        result$error <-paste0(result$error,'\n', e)
-    }
     )
 
+
+    result <- append(result, amelia_result)
+
+
+   mle_result <- list(Bxy.est.mle = NULL,
+                      Bxy.ci.upper.mle = NULL,
+                      Bxy.ci.lower.mle = NULL,
+                      Bzy.est.mle = NULL,
+                      Bzy.ci.upper.mle = NULL,
+                      Bzy.ci.lower.mle = NULL)
+
     tryCatch({
         temp.df <- copy(df)
         temp.df <- temp.df[,x:=x.obs]
-        mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
+        mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula, method='optim')
-        fisher.info <- solve(mod.caroll.lik$hessian)
+        fischer.info <- solve(mod.caroll.lik$hessian)
         coef <- mod.caroll.lik$par
-        ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
+        ci.upper <- coef + sqrt(diag(fischer.info)) * 1.96
-        ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
+        ci.lower <- coef - sqrt(diag(fischer.info)) * 1.96
 
+        mle_result <- list(Bxy.est.mle = coef['x'],
+                           Bxy.ci.upper.mle = ci.upper['x'],
+                           Bxy.ci.lower.mle = ci.lower['x'],
+                           Bzy.est.mle = coef['z'],
+                           Bzy.ci.upper.mle = ci.upper['z'],
+                           Bzy.ci.lower.mle = ci.lower['z'])
+        },
+        error=function(e) {result[['error']] <- as.character(e)
+        })
 
-        result <- append(result,
+    result <- append(result, mle_result)
-                         list(Bxy.est.mle = coef['x'],
+    mle_result_proflik <- list(Bxy.est.mle.profile = NULL,
-                              Bxy.ci.upper.mle = ci.upper['x'],
+                               Bxy.ci.upper.mle.profile = NULL,
-                              Bxy.ci.lower.mle = ci.lower['x'],
+                               Bxy.ci.lower.mle.profile = NULL,
-                              Bzy.est.mle = coef['z'],
+                               Bzy.est.mle.profile = NULL,
-                              Bzy.ci.upper.mle = ci.upper['z'],
+                               Bzy.ci.upper.mle.profile = NULL,
-                              Bzy.ci.lower.mle = ci.lower['z']))
+                               Bzy.ci.lower.mle.profile = NULL)
-    },
-
-    error = function(e){
-        message("An error occurred:\n",e)
-        result$error <- paste0(result$error,'\n', e)
-    })
 
     tryCatch({
+        ## confint_method == 'bbmle'
+        mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula, method='bbmle')
+        coef <- coef(mod.caroll.lik)
+        ci <- confint(mod.caroll.lik, method='spline')
+        ci.lower <- ci[,'2.5 %']
+        ci.upper <- ci[,'97.5 %']
 
-        mod.zhang.lik <- zhang.mle.iv(df)
+        mle_result_proflik <- list(Bxy.est.mle.profile = coef['x'],
-        coef <- coef(mod.zhang.lik)
+                                   Bxy.ci.upper.mle.profile = ci.upper['x'],
-        ci <- confint(mod.zhang.lik,method='quad')
+                                   Bxy.ci.lower.mle.profile = ci.lower['x'],
-        result <- append(result,
+                                   Bzy.est.mle.profile = coef['z'],
-                         list(Bxy.est.zhang = coef['Bxy'],
+                                   Bzy.ci.upper.mle.profile = ci.upper['z'],
-                              Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                                   Bzy.ci.lower.mle.profile = ci.lower['z'])
-                              Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
-                              Bzy.est.zhang = coef['Bzy'],
-                              Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
-                              Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
     },
 
-    error = function(e){
+    error=function(e) {result[['error']] <- as.character(e)
-        message("An error occurred:\n",e)
-        result$error <- paste0(result$error,'\n', e)
     })
         
+    result <- append(result, mle_result_proflik)
+
+    zhang_result <- list(Bxy.est.mle.zhang = NULL,
+                       Bxy.ci.upper.mle.zhang = NULL,
+                       Bxy.ci.lower.mle.zhang = NULL,
+                       Bzy.est.mle.zhang = NULL,
+                       Bzy.ci.upper.mle.zhang = NULL,
+                       Bzy.ci.lower.mle.zhang = NULL)
+
+    tryCatch({
+    mod.zhang.lik <- zhang.mle.iv(df)
+    coef <- coef(mod.zhang.lik)
+    ci <- confint(mod.zhang.lik,method='quad')
+    zhang_result <- list(Bxy.est.zhang = coef['Bxy'],
+                         Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
+                         Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
+                         Bzy.est.zhang = coef['Bzy'],
+                         Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
+                         Bzy.ci.lower.zhang = ci['Bzy','2.5 %'])
+    },
+    error=function(e) {result[['error']] <- as.character(e)
+    })
+    result <- append(result, zhang_result)
+
     ## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model.
     ## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms)
     ## mod.amelia.nok <- zelig(y~x.obs+g, model='ls', data=amelia.out.nok$imputations, cite=FALSE)
@@ -440,29 +459,29 @@ run_simulation <-  function(df, result, outcome_formula=y~x+z, proxy_formula=w_p
                           Bzy.ci.lower.gmm = gmm.res$confint[2,1]))
 
 
-    tryCatch({
+    ## tryCatch({
-    mod.calibrated.mle <- mecor(y ~ MeasError(w_pred, reference = x.obs) + z, df, B=400, method='efficient')
+    ## mod.calibrated.mle <- mecor(y ~ MeasError(w_pred, reference = x.obs) + z, df, B=400, method='efficient')
-    (mod.calibrated.mle)
+    ## (mod.calibrated.mle)
-    (mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',])
+    ## (mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',])
-    result <- append(result, list(
+    ## result <- append(result, list(
-                                 Bxy.est.mecor = mecor.ci['Estimate'],
+    ##                              Bxy.est.mecor = mecor.ci['Estimate'],
-                                 Bxy.ci.upper.mecor = mecor.ci['UCI'],
+    ##                              Bxy.ci.upper.mecor = mecor.ci['UCI'],
-                                 Bxy.ci.lower.mecor = mecor.ci['LCI'])
+    ##                              Bxy.ci.lower.mecor = mecor.ci['LCI'])
-                     )
+    ##                  )
 
-    (mecor.ci <- summary(mod.calibrated.mle)$c$ci['z',])
+    ## (mecor.ci <- summary(mod.calibrated.mle)$c$ci['z',])
 
-    result <- append(result, list(
+    ## result <- append(result, list(
-                                 Bzy.est.mecor = mecor.ci['Estimate'],
+    ##                              Bzy.est.mecor = mecor.ci['Estimate'],
-                                 Bzy.ci.upper.mecor = mecor.ci['UCI'],
+    ##                              Bzy.ci.upper.mecor = mecor.ci['UCI'],
-                                 Bzy.ci.lower.mecor = mecor.ci['LCI'])
+    ##                              Bzy.ci.lower.mecor = mecor.ci['LCI'])
-                     )
+    ##                  )
-    },
+    ## },
-    error = function(e){
+    ## error = function(e){
-        message("An error occurred:\n",e)
+    ##     message("An error occurred:\n",e)
-        result$error <- paste0(result$error, '\n', e)
+    ##     result$error <- paste0(result$error, '\n', e)
-    }
+    ## }
-    )
+    ## )
 ##    clean up memory
 ##    rm(list=c("df","y","x","g","w","v","train","p","amelia.out.k","amelia.out.nok", "mod.calibrated.mle","gmm.res","mod.amelia.k","mod.amelia.nok", "model.true","model.naive","model.feasible"))
     

simulations/summarize_estimator.R

@@ -0,0 +1,58 @@
+library(ggdist)
+
+summarize.estimator <- function(sims.df, suffix='naive', coefname='x'){
+
+    reported_vars <- c(
+                       'Bxy',
+                       paste0('B',coefname,'y.est.',suffix),
+                       paste0('B',coefname,'y.ci.lower.',suffix),
+                       paste0('B',coefname,'y.ci.upper.',suffix)
+                       )
+
+    
+    grouping_vars <- c('N','m','B0', 'Bxy', 'Bzy', 'Bzx', 'Px', 'Py','y_explained_variance', 'prediction_accuracy','outcome_formula','proxy_formula','truth_formula','z_bias','y_bias')
+
+    grouping_vars <- grouping_vars[grouping_vars %in% names(df)]
+
+    part <- sims.df[,
+                    unique(c(reported_vars,
+                             grouping_vars)),
+                    with=FALSE]
+
+
+    true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
+    zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
+    bias <- part[[paste0('B',coefname,'y')]] - part[[paste0('B',coefname,'y.est.',suffix)]]
+    sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
+
+    part <- part[,':='(true.in.ci = true.in.ci,
+                       zero.in.ci = zero.in.ci,
+                       bias=bias,
+                       sign.correct =sign.correct)]
+
+
+    part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
+                          mean.bias = mean(bias),
+                          mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]],na.rm=T),
+                          var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]],na.rm=T),
+                          est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.975,na.rm=T),
+                          est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.025,na.rm=T),
+                          mean.ci.upper = mean(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],na.rm=T),
+                          mean.ci.lower = mean(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],na.rm=T),
+                          median.ci.upper = median(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],na.rm=T),
+                          median.ci.lower = median(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],na.rm=T),
+                          ci.upper.975 = quantile(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],0.975,na.rm=T),
+                          ci.upper.025 = quantile(.SD[[paste0('B',coefname,'y.ci.upper.',suffix)]],0.025,na.rm=T),
+                          ci.lower.975 = quantile(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],0.975,na.rm=T),
+                          ci.lower.025 = quantile(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]],0.025,na.rm=T),
+                          N.ci.is.NA = sum(is.na(.SD[[paste0('B',coefname,'y.ci.lower.',suffix)]])),
+                          N.sims = .N,
+                          p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
+                          variable=coefname,
+                          method=suffix
+                          ),
+                      by=grouping_vars,
+                      ]
+    
+    return(part.plot)
+}