update simulation code for examples 1-3
This commit is contained in:
parent
cb3f850c24
commit
46e2d1fe48
@ -1,8 +1,10 @@
|
||||
### EXAMPLE 2_b: demonstrates how measurement error can lead to a type sign error in a covariate
|
||||
### This is the same as example 2, only instead of x->k we have k->x.
|
||||
### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
|
||||
### Even when you include the proxy variable in the regression.
|
||||
### But with some ground truth and multiple imputation, you can fix it.
|
||||
### EXAMPLE 2_b: demonstrates how measurement error can lead to a type
|
||||
### sign error in a covariate This is the same as example 2, only
|
||||
### instead of x->k we have k->x. Even when you have a good
|
||||
### predictor, if it's biased against a covariate you can get the
|
||||
### wrong sign. Even when you include the proxy variable in the
|
||||
### regression. But with some ground truth and multiple imputation,
|
||||
### you can fix it.
|
||||
|
||||
library(argparser)
|
||||
library(mecor)
|
||||
@ -12,9 +14,9 @@ library(filelock)
|
||||
library(arrow)
|
||||
library(Amelia)
|
||||
library(Zelig)
|
||||
|
||||
library(predictionError)
|
||||
options(amelia.parallel="no",
|
||||
amelia.ncpus=1)
|
||||
options(amelia.parallel="no", amelia.ncpus=1)
|
||||
|
||||
source("simulation_base.R")
|
||||
|
||||
@ -28,20 +30,18 @@ source("simulation_base.R")
|
||||
#### how much power do we get from the model in the first place? (sweeping N and m)
|
||||
####
|
||||
|
||||
simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_variance=0.025, gx_explained_variance=0.15, prediction_accuracy=0.73, seed=1){
|
||||
simulate_data <- function(N, m, B0=0, Bxy=0.2, Bzy=-0.2, Bzx=0.2, y_explained_variance=0.025, prediction_accuracy=0.73, seed=1){
|
||||
set.seed(seed)
|
||||
g <- rbinom(N, 1, 0.5)
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
# x.var.epsilon <- var(Bzx *z) * ((1-zx_explained_variance)/zx_explained_variance)
|
||||
xprime <- Bzx * z #+ x.var.epsilon
|
||||
x <- rbinom(N,1,plogis(xprime))
|
||||
|
||||
x.var.epsilon <- var(Bgx *g) * ((1-gx_explained_variance)/gx_explained_variance)
|
||||
x.epsilon <- rnorm(N,sd=sqrt(x.var.epsilon))
|
||||
xprime <- Bgx * g + x.epsilon
|
||||
x <- as.integer(logistic(scale(xprime)) > 0.5)
|
||||
|
||||
y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bxy*x,Bgy*g)) * ((1-y_explained_variance)/y_explained_variance)
|
||||
y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bxy*x,Bzy*z)) * ((1-y_explained_variance)/y_explained_variance)
|
||||
y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
|
||||
y <- Bgy * g + Bxy * x + y.epsilon
|
||||
y <- Bzy * z + Bxy * x + y.epsilon
|
||||
|
||||
df <- data.table(x=x,xprime=xprime,y=y,g=g)
|
||||
df <- data.table(x=x,y=y,z=z)
|
||||
|
||||
if(m < N){
|
||||
df <- df[sample(nrow(df), m), x.obs := x]
|
||||
@ -49,42 +49,53 @@ simulate_data <- function(N, m, B0=0, Bxy=0.2, Bgy=-0.2, Bgx=0.2, y_explained_va
|
||||
df <- df[, x.obs := x]
|
||||
}
|
||||
|
||||
df <- df[,w_pred:=x]
|
||||
df <- df[sample(1:N,(1-prediction_accuracy)*N),w_pred:=(w_pred-1)**2]
|
||||
w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response')
|
||||
df <- df[,':='(w=w, w_pred = w_pred)]
|
||||
## how can you make a model with a specific accuracy?
|
||||
w0 =(1-x)**2 + (-1)**(1-x) * prediction_accuracy
|
||||
|
||||
## how can you make a model with a specific accuracy, with a continuous latent variable.
|
||||
# now it makes the same amount of mistake to each point, probably
|
||||
# add mean0 noise to the odds.
|
||||
|
||||
w.noisey.odds = rlogis(N,qlogis(w0))
|
||||
df[,w := plogis(w.noisey.odds)]
|
||||
df[,w_pred:=as.integer(w > 0.5)]
|
||||
(mean(df$x==df$w_pred))
|
||||
return(df)
|
||||
}
|
||||
|
||||
parser <- arg_parser("Simulate data and fit corrected models")
|
||||
parser <- add_argument(parser, "--N", default=500, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=100, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=57, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_1.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
|
||||
parser <- add_argument(parser, "--gx_explained_variance", help='what proportion of the variance of x can be explained by g?', default=0.15)
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05)
|
||||
# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
|
||||
|
||||
parser <- add_argument(parser, "--Bzx", help='coefficient of z on x?', default=1)
|
||||
args <- parse_args(parser)
|
||||
|
||||
B0 <- 0
|
||||
Bxy <- 0.2
|
||||
Bgy <- -0.2
|
||||
Bgx <- 0.4
|
||||
Bxy <- 0.3
|
||||
Bzy <- -0.3
|
||||
Bzx <- args$Bzx
|
||||
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, Bgx, seed=args$seed, y_explained_variance = args$y_explained_variance, gx_explained_variance = args$gx_explained_variance, prediction_accuracy=args$prediction_accuracy)
|
||||
if (args$m < args$N){
|
||||
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'Bgx'=Bgx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'gx_explained_variance' = args$gx_explained_variance, "prediction_accuracy"=args$prediction_accuracy)
|
||||
outline <- run_simulation(df, result)
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance, prediction_accuracy=args$prediction_accuracy)
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline))
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'zx_explained_variance' = args$zx_explained_variance, "prediction_accuracy"=args$prediction_accuracy, "error"="")
|
||||
|
||||
outline <- run_simulation(df, result)
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
|
||||
@ -31,17 +31,17 @@ source("simulation_base.R")
|
||||
|
||||
## one way to do it is by adding correlation to x.obs and y that isn't in w.
|
||||
## in other words, the model is missing an important feature of x.obs that's related to y.
|
||||
simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
|
||||
set.seed(seed)
|
||||
# make w and y dependent
|
||||
g <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, 0.5)
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, Bzx * z + 0.5)
|
||||
|
||||
y.var.epsilon <- (var(Bgy * g) + var(Bxy *x) + 2*cov(Bgy*g,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
|
||||
y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
|
||||
y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
|
||||
y <- Bgy * g + Bxy * x + y.epsilon
|
||||
y <- Bzy * z + Bxy * x + y.epsilon
|
||||
|
||||
df <- data.table(x=x,y=y,g=g)
|
||||
df <- data.table(x=x,y=y,z=z)
|
||||
|
||||
if(m < N){
|
||||
df <- df[sample(nrow(df), m), x.obs := x]
|
||||
@ -49,61 +49,117 @@ simulate_data <- function(N, m, B0, Bxy, Bgy, seed, y_explained_variance=0.025,
|
||||
df <- df[, x.obs := x]
|
||||
}
|
||||
|
||||
df <- df[,w_pred:=x]
|
||||
## px <- mean(x)
|
||||
## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
|
||||
|
||||
pg <- mean(g)
|
||||
px <- mean(x)
|
||||
## # this works because of conditional probability
|
||||
## accuracy_x0 <- prediction_accuracy / (px*(accuracy_imbalance_ratio) + (1-px))
|
||||
## accuracy_x1 <- accuracy_imbalance_ratio * accuracy_x0
|
||||
|
||||
## x0 <- df[x==0]$x
|
||||
## x1 <- df[x==1]$x
|
||||
## nx1 <- nrow(df[x==1])
|
||||
## nx0 <- nrow(df[x==0])
|
||||
|
||||
## yx0 <- df[x==0]$y
|
||||
## yx1 <- df[x==1]$y
|
||||
|
||||
# tranform yz0.1 into a logistic distribution with mean accuracy_z0
|
||||
## acc.x0 <- plogis(0.5*scale(yx0) + qlogis(accuracy_x0))
|
||||
## acc.x1 <- plogis(1.5*scale(yx1) + qlogis(accuracy_x1))
|
||||
|
||||
## w0x0 <- (1-x0)**2 + (-1)**(1-x0) * acc.x0
|
||||
## w0x1 <- (1-x1)**2 + (-1)**(1-x1) * acc.x1
|
||||
pz <- mean(z)
|
||||
accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
|
||||
|
||||
# this works because of conditional probability
|
||||
accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg))
|
||||
accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0
|
||||
accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
|
||||
accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
|
||||
|
||||
dfg0 <- df[g==0]
|
||||
ng0 <- nrow(dfg0)
|
||||
dfg1 <- df[g==1]
|
||||
ng1 <- nrow(dfg1)
|
||||
z0x0 <- df[(z==0) & (x==0)]$x
|
||||
z0x1 <- df[(z==0) & (x==1)]$x
|
||||
z1x0 <- df[(z==1) & (x==0)]$x
|
||||
z1x1 <- df[(z==1) & (x==1)]$x
|
||||
|
||||
dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2]
|
||||
dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2]
|
||||
yz0x0 <- df[(z==0) & (x==0)]$y
|
||||
yz0x1 <- df[(z==0) & (x==1)]$y
|
||||
yz1x0 <- df[(z==1) & (x==0)]$y
|
||||
yz1x1 <- df[(z==1) & (x==1)]$y
|
||||
|
||||
df <- rbind(dfg0,dfg1)
|
||||
nz0x0 <- nrow(df[(z==0) & (x==0)])
|
||||
nz0x1 <- nrow(df[(z==0) & (x==1)])
|
||||
nz1x0 <- nrow(df[(z==1) & (x==0)])
|
||||
nz1x1 <- nrow(df[(z==1) & (x==1)])
|
||||
|
||||
w <- predict(glm(x ~ w_pred,data=df,family=binomial(link='logit')),type='response')
|
||||
df <- df[,':='(w=w, w_pred = w_pred)]
|
||||
yz1 <- df[z==1]$y
|
||||
yz1 <- df[z==1]$y
|
||||
|
||||
# tranform yz0.1 into a logistic distribution with mean accuracy_z0
|
||||
acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
|
||||
acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
|
||||
acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
|
||||
acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
|
||||
|
||||
w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
|
||||
w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
|
||||
w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
|
||||
w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
|
||||
|
||||
##perrorz0 <- w0z0*(pyz0)
|
||||
##perrorz1 <- w0z1*(pyz1)
|
||||
|
||||
w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
|
||||
w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
|
||||
w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
|
||||
w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
|
||||
|
||||
df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
|
||||
df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]
|
||||
df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]
|
||||
df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]
|
||||
|
||||
df[,w_pred:=as.integer(w > 0.5)]
|
||||
print(mean(df[z==0]$x == df[z==0]$w_pred))
|
||||
print(mean(df[z==1]$x == df[z==1]$w_pred))
|
||||
print(mean(df$w_pred == df$x))
|
||||
return(df)
|
||||
}
|
||||
|
||||
parser <- arg_parser("Simulate data and fit corrected models")
|
||||
parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=432, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--N", default=1400, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=50, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
|
||||
parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
|
||||
|
||||
|
||||
args <- parse_args(parser)
|
||||
|
||||
B0 <- 0
|
||||
Bxy <- 0.2
|
||||
Bgy <- -0.2
|
||||
Bxy <- 0.3
|
||||
Bzy <- args$Bzy
|
||||
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)
|
||||
if(args$m < args$N){
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, args$Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, args$accuracy_imbalance_difference)
|
||||
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, error='')
|
||||
|
||||
outline <- run_simulation_depvar(df=df, result)
|
||||
outline <- run_simulation(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x+z+y+x:y, truth_formula=x~z)
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
}
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline))
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
|
||||
@ -31,18 +31,18 @@ source("simulation_base.R")
|
||||
|
||||
## one way to do it is by adding correlation to x.obs and y that isn't in w.
|
||||
## in other words, the model is missing an important feature of x.obs that's related to y.
|
||||
simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
|
||||
set.seed(seed)
|
||||
# make w and y dependent
|
||||
g <- rbinom(N, 1, 0.5)
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, 0.5)
|
||||
|
||||
ystar <- Bgy * g + Bxy * x
|
||||
y <- rbinom(N,1,logistic(ystar))
|
||||
ystar <- Bzy * z + Bxy * x
|
||||
y <- rbinom(N,1,plogis(ystar))
|
||||
|
||||
# glm(y ~ x + g, family="binomial")
|
||||
# glm(y ~ x + z, family="binomial")
|
||||
|
||||
df <- data.table(x=x,y=y,ystar=ystar,g=g)
|
||||
df <- data.table(x=x,y=y,ystar=ystar,z=z)
|
||||
|
||||
if(m < N){
|
||||
df <- df[sample(nrow(df), m), y.obs := y]
|
||||
@ -52,36 +52,44 @@ simulate_data <- function(N, m, B0, Bxy, Bgy, seed, prediction_accuracy=0.73, ac
|
||||
|
||||
df <- df[,w_pred:=y]
|
||||
|
||||
pg <- mean(g)
|
||||
pz <- mean(z)
|
||||
|
||||
accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
|
||||
|
||||
# this works because of conditional probability
|
||||
accuracy_g0 <- prediction_accuracy / (pg*(accuracy_imbalance_ratio) + (1-pg))
|
||||
accuracy_g1 <- accuracy_imbalance_ratio * accuracy_g0
|
||||
accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
|
||||
accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
|
||||
|
||||
dfg0 <- df[g==0]
|
||||
ng0 <- nrow(dfg0)
|
||||
dfg1 <- df[g==1]
|
||||
ng1 <- nrow(dfg1)
|
||||
|
||||
dfg0 <- dfg0[sample(ng0, (1-accuracy_g0)*ng0), w_pred := (w_pred-1)**2]
|
||||
dfg1 <- dfg1[sample(ng1, (1-accuracy_g1)*ng1), w_pred := (w_pred-1)**2]
|
||||
yz0 <- df[z==0]$y
|
||||
yz1 <- df[z==1]$y
|
||||
nz1 <- nrow(df[z==1])
|
||||
nz0 <- nrow(df[z==0])
|
||||
|
||||
df <- rbind(dfg0,dfg1)
|
||||
acc_z0 <- plogis(0.7*scale(yz0) + qlogis(accuracy_z0))
|
||||
acc_z1 <- plogis(1.3*scale(yz1) + qlogis(accuracy_z1))
|
||||
|
||||
wmod <- glm(y.obs ~ w_pred,data=df[!is.null(y.obs)],family=binomial(link='logit'))
|
||||
w <- predict(wmod,df,type='response')
|
||||
w0z0 <- (1-yz0)**2 + (-1)**(1-yz0) * acc_z0
|
||||
w0z1 <- (1-yz1)**2 + (-1)**(1-yz1) * acc_z1
|
||||
|
||||
df <- df[,':='(w=w)]
|
||||
w0z0.noisy.odds <- rlogis(nz0,qlogis(w0z0))
|
||||
w0z1.noisy.odds <- rlogis(nz1,qlogis(w0z1))
|
||||
df[z==0,w:=plogis(w0z0.noisy.odds)]
|
||||
df[z==1,w:=plogis(w0z1.noisy.odds)]
|
||||
|
||||
df[,w_pred:=as.integer(w > 0.5)]
|
||||
|
||||
print(mean(df[y==0]$y == df[y==0]$w_pred))
|
||||
print(mean(df[y==1]$y == df[y==1]$w_pred))
|
||||
print(mean(df$w_pred == df$y))
|
||||
|
||||
return(df)
|
||||
}
|
||||
|
||||
parser <- arg_parser("Simulate data and fit corrected models")
|
||||
parser <- add_argument(parser, "--N", default=5000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=200, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=4321, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
|
||||
@ -90,24 +98,26 @@ parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much
|
||||
args <- parse_args(parser)
|
||||
|
||||
B0 <- 0
|
||||
Bxy <- 0.2
|
||||
Bgy <- -0.2
|
||||
Bxy <- 0.7
|
||||
Bzy <- -0.7
|
||||
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bgy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
|
||||
if(args$m < args$N){
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
|
||||
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bgy'=Bgy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
|
||||
|
||||
outline <- run_simulation_depvar(df=df, result)
|
||||
outline <- run_simulation_depvar(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ y*x + y*z + z*x)
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline))
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
|
||||
@ -1,28 +1,42 @@
|
||||
|
||||
SHELL=bash
|
||||
|
||||
Ns=[500,1000,10000]
|
||||
ms=[50, 100, 250, 500]
|
||||
Ns=[1000,3600,14400]
|
||||
ms=[75,150,300]
|
||||
seeds=[$(shell seq -s, 1 250)]
|
||||
explained_variances=[0.1]
|
||||
|
||||
all:remembr.RDS
|
||||
|
||||
srun=srun -A comdata -p compute-bigmem --time=10:00:00 --mem 4G -c 1
|
||||
srun=srun -A comdata -p compute-bigmem --time=6:00:00 --mem 4G -c 1
|
||||
|
||||
example_1_jobs: 01_two_covariates.R
|
||||
grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"]}' --outfile example_1_jobs
|
||||
|
||||
joblists:example_1_jobs example_2_jobs example_3_jobs
|
||||
|
||||
# test_true_z_jobs: test_true_z.R simulation_base.R
|
||||
# grid_sweep.py --command "Rscript test_true_z.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["test_true_z.feather"], "y_explained_variancevari":${explained_variances}, "Bzx":${Bzx}}' --outfile test_true_z_jobsb
|
||||
|
||||
# test_true_z.feather: test_true_z_jobs
|
||||
# rm -f test_true_z.feather
|
||||
# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 test_true_z_jobs
|
||||
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 test_true_z_jobs
|
||||
|
||||
|
||||
example_1_jobs: 01_two_covariates.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 01_two_covariates.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_1.feather"], "y_explained_variance":${explained_variances}, "Bzx":[0.1]}' --outfile example_1_jobs
|
||||
|
||||
example_1.feather: example_1_jobs
|
||||
rm -f example_1.feather
|
||||
sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_1_jobs
|
||||
sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
|
||||
sbatch --wait --verbose --array=1-$(shell cat example_1_jobs | wc -l) run_simulation.sbatch 0 example_1_jobs
|
||||
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
|
||||
|
||||
example_2_jobs: example_2.R
|
||||
grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"]}' --outfile example_2_jobs
|
||||
example_2_jobs: 02_indep_differential.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "accuracy_imbalance_difference":[0.3], "Bzy":[0.3]}' --outfile example_2_jobs
|
||||
|
||||
example_2.feather: example_2_jobs
|
||||
rm -f example_2.feather
|
||||
sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_jobs
|
||||
sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
|
||||
sbatch --wait --verbose --array=1-$(shell cat example_2_jobs | wc -l) run_simulation.sbatch 0 example_2_jobs
|
||||
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_jobs
|
||||
|
||||
# example_2_B_jobs: example_2_B.R
|
||||
# grid_sweep.py --command "Rscript example_2_B.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2_B.feather"]}' --outfile example_2_B_jobs
|
||||
@ -31,23 +45,24 @@ example_2.feather: example_2_jobs
|
||||
# rm -f example_2_B.feather
|
||||
# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
|
||||
|
||||
example_3_jobs: 03_depvar_differential.R
|
||||
grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"]}' --outfile example_3_jobs
|
||||
example_3_jobs: 03_depvar_differential.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
|
||||
|
||||
example_3.feather: example_3_jobs
|
||||
rm -f example_3.feather
|
||||
sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_3_jobs
|
||||
sbatch --wait --verbose --array=3001-6000 run_simulation.sbatch 0 example_3_jobs
|
||||
sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs
|
||||
|
||||
remembr.RDS:example_1.feather example_2.feather example_3.feather
|
||||
|
||||
remembr.RDS:example_1.feather example_2.feather example_3.feather plot_example.R plot_dv_example.R
|
||||
rm -f remembr.RDS
|
||||
${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
|
||||
${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
|
||||
${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
|
||||
|
||||
clean:
|
||||
rm *.feather
|
||||
rm remembr.RDS
|
||||
|
||||
rm -f remembr.RDS
|
||||
rm -f example_*_jobs
|
||||
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_2_B_jobs
|
||||
|
||||
# example_2_B_mecor_jobs:
|
||||
|
||||
227
simulations/measerr_methods.R
Normal file
227
simulations/measerr_methods.R
Normal file
@ -0,0 +1,227 @@
|
||||
library(formula.tools)
|
||||
library(matrixStats)
|
||||
|
||||
## df: dataframe to model
|
||||
## outcome_formula: formula for y | x, z
|
||||
## outcome_family: family for y | x, z
|
||||
## proxy_formula: formula for w | x, z, y
|
||||
## proxy_family: family for w | x, z, y
|
||||
## truth_formula: formula for x | z
|
||||
## truth_family: family for x | z
|
||||
|
||||
### ideal formulas for example 1
|
||||
# test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
|
||||
|
||||
### ideal formulas for example 2
|
||||
# test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
|
||||
|
||||
|
||||
## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y
|
||||
measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit')){
|
||||
|
||||
nll <- function(params){
|
||||
df.obs <- model.frame(outcome_formula, df)
|
||||
proxy.variable <- all.vars(proxy_formula)[1]
|
||||
proxy.model.matrix <- model.matrix(proxy_formula, df)
|
||||
response.var <- all.vars(outcome_formula)[1]
|
||||
y.obs <- with(df.obs,eval(parse(text=response.var)))
|
||||
outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
|
||||
|
||||
param.idx <- 1
|
||||
n.outcome.model.covars <- dim(outcome.model.matrix)[2]
|
||||
outcome.params <- params[param.idx:n.outcome.model.covars]
|
||||
param.idx <- param.idx + n.outcome.model.covars
|
||||
|
||||
if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
|
||||
ll.y.obs <- vector(mode='numeric', length=length(y.obs))
|
||||
ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
|
||||
ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
|
||||
}
|
||||
|
||||
df.obs <- model.frame(proxy_formula,df)
|
||||
n.proxy.model.covars <- dim(proxy.model.matrix)[2]
|
||||
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
|
||||
|
||||
param.idx <- param.idx + n.proxy.model.covars
|
||||
proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
|
||||
|
||||
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
||||
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
|
||||
ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
|
||||
ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
|
||||
ll.obs <- sum(ll.y.obs + ll.w.obs)
|
||||
|
||||
df.unobs <- df[is.na(df[[response.var]])]
|
||||
df.unobs.y1 <- copy(df.unobs)
|
||||
df.unobs.y1[[response.var]] <- 1
|
||||
df.unobs.y0 <- copy(df.unobs)
|
||||
df.unobs.y0[[response.var]] <- 1
|
||||
|
||||
## integrate out y
|
||||
outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
|
||||
|
||||
if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
|
||||
ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
|
||||
ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
|
||||
ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
|
||||
ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
|
||||
}
|
||||
|
||||
proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
|
||||
proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
|
||||
proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
|
||||
|
||||
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
||||
ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
|
||||
ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
|
||||
ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
|
||||
ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
|
||||
ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
|
||||
ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
|
||||
ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
|
||||
ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
|
||||
ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
|
||||
ll <- ll.unobs + ll.obs
|
||||
return(-ll)
|
||||
}
|
||||
|
||||
params <- colnames(model.matrix(outcome_formula,df))
|
||||
lower <- rep(-Inf, length(params))
|
||||
proxy.params <- colnames(model.matrix(proxy_formula, df))
|
||||
params <- c(params, paste0('proxy_',proxy.params))
|
||||
lower <- c(lower, rep(-Inf, length(proxy.params)))
|
||||
start <- rep(0.1,length(params))
|
||||
names(start) <- params
|
||||
|
||||
fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
|
||||
return(fit)
|
||||
}
|
||||
|
||||
measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit')){
|
||||
|
||||
measrr_mle_nll <- function(params){
|
||||
df.obs <- model.frame(outcome_formula, df)
|
||||
|
||||
proxy.variable <- all.vars(proxy_formula)[1]
|
||||
proxy.model.matrix <- model.matrix(proxy_formula, df)
|
||||
|
||||
response.var <- all.vars(outcome_formula)[1]
|
||||
y.obs <- with(df.obs,eval(parse(text=response.var)))
|
||||
|
||||
outcome.model.matrix <- model.matrix(outcome_formula, df)
|
||||
|
||||
param.idx <- 1
|
||||
n.outcome.model.covars <- dim(outcome.model.matrix)[2]
|
||||
outcome.params <- params[param.idx:n.outcome.model.covars]
|
||||
param.idx <- param.idx + n.outcome.model.covars
|
||||
|
||||
## likelihood for the fully observed data
|
||||
if(outcome_family$family == "gaussian"){
|
||||
sigma.y <- params[param.idx]
|
||||
param.idx <- param.idx + 1
|
||||
ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
|
||||
}
|
||||
|
||||
df.obs <- model.frame(proxy_formula,df)
|
||||
n.proxy.model.covars <- dim(proxy.model.matrix)[2]
|
||||
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
|
||||
param.idx <- param.idx + n.proxy.model.covars
|
||||
proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
|
||||
|
||||
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
||||
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
|
||||
ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
|
||||
ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
|
||||
df.obs <- model.frame(truth_formula, df)
|
||||
truth.variable <- all.vars(truth_formula)[1]
|
||||
truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
|
||||
truth.model.matrix <- model.matrix(truth_formula,df)
|
||||
n.truth.model.covars <- dim(truth.model.matrix)[2]
|
||||
|
||||
truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
|
||||
|
||||
if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
|
||||
ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
|
||||
ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
|
||||
ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
|
||||
ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
|
||||
|
||||
## likelihood for the predicted data
|
||||
## integrate out the "truth" variable.
|
||||
|
||||
if(truth_family$family=='binomial'){
|
||||
df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
|
||||
df.unobs.x1 <- copy(df.unobs)
|
||||
df.unobs.x1[,'x'] <- 1
|
||||
df.unobs.x0 <- copy(df.unobs)
|
||||
df.unobs.x0[,'x'] <- 0
|
||||
outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
|
||||
|
||||
outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
|
||||
outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
|
||||
if(outcome_family$family=="gaussian"){
|
||||
ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
|
||||
ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
|
||||
}
|
||||
|
||||
if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
|
||||
|
||||
proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
|
||||
proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
|
||||
proxy.unobs <- df.unobs[[proxy.variable]]
|
||||
ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
|
||||
ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
|
||||
|
||||
ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
|
||||
ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
|
||||
|
||||
ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
|
||||
ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
|
||||
}
|
||||
|
||||
if(truth_family$link=='logit'){
|
||||
truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
|
||||
ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
|
||||
ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
}
|
||||
|
||||
ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
|
||||
ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
|
||||
ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
|
||||
return(-(ll.unobs + ll.obs))
|
||||
}
|
||||
|
||||
outcome.params <- colnames(model.matrix(outcome_formula,df))
|
||||
lower <- rep(-Inf, length(outcome.params))
|
||||
|
||||
if(outcome_family$family=='gaussian'){
|
||||
params <- c(outcome.params, 'sigma_y')
|
||||
lower <- c(lower, 0.00001)
|
||||
} else {
|
||||
params <- outcome.params
|
||||
}
|
||||
|
||||
proxy.params <- colnames(model.matrix(proxy_formula, df))
|
||||
params <- c(params, paste0('proxy_',proxy.params))
|
||||
lower <- c(lower, rep(-Inf, length(proxy.params)))
|
||||
|
||||
truth.params <- colnames(model.matrix(truth_formula, df))
|
||||
params <- c(params, paste0('truth_', truth.params))
|
||||
lower <- c(lower, rep(-Inf, length(truth.params)))
|
||||
start <- rep(0.1,length(params))
|
||||
names(start) <- params
|
||||
|
||||
fit <- optim(start, fn = measrr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
|
||||
|
||||
return(fit)
|
||||
}
|
||||
@ -10,204 +10,73 @@ parser <- add_argument(parser, "--infile", default="", help="name of the file to
|
||||
parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
|
||||
args <- parse_args(parser)
|
||||
|
||||
|
||||
|
||||
summarize.estimator <- function(df, suffix='naive', coefname='x'){
|
||||
|
||||
part <- df[,c('N',
|
||||
'm',
|
||||
'Bxy',
|
||||
paste0('B',coefname,'y.est.',suffix),
|
||||
paste0('B',coefname,'y.ci.lower.',suffix),
|
||||
paste0('B',coefname,'y.ci.upper.',suffix),
|
||||
'y_explained_variance',
|
||||
'Bzy',
|
||||
'accuracy_imbalance_difference'
|
||||
),
|
||||
with=FALSE]
|
||||
|
||||
true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
|
||||
zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
|
||||
bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
|
||||
sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
|
||||
|
||||
part <- part[,':='(true.in.ci = true.in.ci,
|
||||
zero.in.ci = zero.in.ci,
|
||||
bias=bias,
|
||||
sign.correct =sign.correct)]
|
||||
|
||||
part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
|
||||
var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
|
||||
est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95),
|
||||
est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable=coefname,
|
||||
method=suffix
|
||||
),
|
||||
by=c("N","m",'Bzy','accuracy_imbalance_difference','y_explained_variance')
|
||||
]
|
||||
|
||||
return(part.plot)
|
||||
}
|
||||
|
||||
|
||||
build_plot_dataset <- function(df){
|
||||
x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)]
|
||||
x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive),
|
||||
bias = Bxy - Bxy.est.naive,
|
||||
Bxy.est.naive = Bxy.est.naive,
|
||||
sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))]
|
||||
|
||||
x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bxy.est.naive),
|
||||
var.est = var(Bxy.est.naive),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='x',
|
||||
method='Naive'
|
||||
),
|
||||
by=c('N','m')]
|
||||
x.true <- summarize.estimator(df, 'true','x')
|
||||
z.true <- summarize.estimator(df, 'true','z')
|
||||
|
||||
x.naive <- summarize.estimator(df, 'naive','x')
|
||||
z.naive <- summarize.estimator(df, 'naive','z')
|
||||
|
||||
g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)]
|
||||
g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive),
|
||||
bias = Bgy - Bgy.est.naive,
|
||||
Bgy.est.naive = Bgy.est.naive,
|
||||
sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))]
|
||||
x.feasible <- summarize.estimator(df, 'feasible','x')
|
||||
z.feasible <- summarize.estimator(df, 'feasible','z')
|
||||
|
||||
g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bgy.est.naive),
|
||||
var.est = var(Bgy.est.naive),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='g',
|
||||
method='Naive'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)]
|
||||
x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible),
|
||||
bias = Bxy - Bxy.est.feasible,
|
||||
Bxy.est.feasible = Bxy.est.feasible,
|
||||
sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))]
|
||||
|
||||
x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bxy.est.feasible),
|
||||
var.est = var(Bxy.est.feasible),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='x',
|
||||
method='Feasible'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)]
|
||||
g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible),
|
||||
bias = Bgy - Bgy.est.feasible,
|
||||
Bgy.est.feasible = Bgy.est.feasible,
|
||||
sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))]
|
||||
|
||||
g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bgy.est.feasible),
|
||||
var.est = var(Bgy.est.feasible),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='g',
|
||||
method='Feasible'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
|
||||
x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)]
|
||||
|
||||
x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full),
|
||||
bias = Bxy.est.true - Bxy.est.amelia.full,
|
||||
sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))]
|
||||
|
||||
x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bxy.est.amelia.full),
|
||||
var.est = var(Bxy.est.amelia.full),
|
||||
N.sims = .N,
|
||||
variable='x',
|
||||
method='Multiple imputation'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)]
|
||||
g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full),
|
||||
bias = Bgy.est.amelia.full - Bgy.est.true,
|
||||
sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))]
|
||||
|
||||
g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bgy.est.amelia.full),
|
||||
var.est = var(Bgy.est.amelia.full),
|
||||
N.sims = .N,
|
||||
variable='g',
|
||||
method='Multiple imputation'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
x.mle <- df[,.(N,m, Bxy.est.true, Bxy.est.mle, Bxy.ci.lower.mle, Bxy.ci.upper.mle)]
|
||||
|
||||
x.mle <- x.mle[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.mle) & (Bxy.est.true <= Bxy.ci.upper.mle),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.mle) & (0 <= Bxy.ci.upper.mle),
|
||||
bias = Bxy.est.mle - Bxy.est.true,
|
||||
sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mle))]
|
||||
|
||||
x.mle.plot <- x.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bxy.est.mle),
|
||||
var.est = var(Bxy.est.mle),
|
||||
N.sims = .N,
|
||||
variable='x',
|
||||
method='Maximum Likelihood'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
|
||||
g.mle <- df[,.(N,m, Bgy.est.true, Bgy.est.mle, Bgy.ci.lower.mle, Bgy.ci.upper.mle)]
|
||||
|
||||
g.mle <- g.mle[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mle) & (Bgy.est.true <= Bgy.ci.upper.mle),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.mle) & (0 <= Bgy.ci.upper.mle),
|
||||
bias = Bgy.est.mle - Bgy.est.true,
|
||||
sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mle))]
|
||||
|
||||
g.mle.plot <- g.mle[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bgy.est.mle),
|
||||
var.est = var(Bgy.est.mle),
|
||||
N.sims = .N,
|
||||
variable='g',
|
||||
method='Maximum Likelihood'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
|
||||
|
||||
x.pseudo <- df[,.(N,m, Bxy.est.true, Bxy.est.pseudo, Bxy.ci.lower.pseudo, Bxy.ci.upper.pseudo)]
|
||||
|
||||
x.pseudo <- x.pseudo[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.pseudo) & (Bxy.est.true <= Bxy.ci.upper.pseudo),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.pseudo) & (0 <= Bxy.ci.upper.pseudo),
|
||||
bias = Bxy.est.pseudo - Bxy.est.true,
|
||||
sign.correct = sign(Bxy.est.true) == sign(Bxy.est.pseudo))]
|
||||
|
||||
x.pseudo.plot <- x.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bxy.est.pseudo),
|
||||
var.est = var(Bxy.est.pseudo),
|
||||
N.sims = .N,
|
||||
variable='x',
|
||||
method='Pseudo Likelihood'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
|
||||
|
||||
g.pseudo <- df[,.(N,m, Bgy.est.true, Bgy.est.pseudo, Bgy.ci.lower.pseudo, Bgy.ci.upper.pseudo)]
|
||||
|
||||
g.pseudo <- g.pseudo[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.pseudo) & (Bgy.est.true <= Bgy.ci.upper.pseudo),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.pseudo) & (0 <= Bgy.ci.upper.pseudo),
|
||||
bias = Bgy.est.pseudo - Bgy.est.true,
|
||||
sign.correct = sign(Bgy.est.true) == sign(Bgy.est.pseudo))]
|
||||
|
||||
g.pseudo.plot <- g.pseudo[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bgy.est.pseudo),
|
||||
var.est = var(Bgy.est.pseudo),
|
||||
N.sims = .N,
|
||||
variable='g',
|
||||
method='Pseudo Likelihood'
|
||||
),
|
||||
by=c('N','m')]
|
||||
x.amelia.full <- summarize.estimator(df, 'amelia.full','x')
|
||||
z.amelia.full <- summarize.estimator(df, 'amelia.full','z')
|
||||
|
||||
x.mle <- summarize.estimator(df, 'mle','x')
|
||||
z.mle <- summarize.estimator(df, 'mle','z')
|
||||
|
||||
x.zhang <- summarize.estimator(df, 'zhang','x')
|
||||
z.zhang <- summarize.estimator(df, 'zhang','z')
|
||||
|
||||
accuracy <- df[,mean(accuracy)]
|
||||
|
||||
plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mle.plot, g.mle.plot, x.pseudo.plot, g.pseudo.plot, x.feasible.plot, g.feasible.plot),use.names=T)
|
||||
plot.df <- rbindlist(list(x.true, z.true, x.naive,z.naive,x.amelia.full,z.amelia.full,x.mle, z.mle, x.zhang, z.zhang, x.feasible, z.feasible),use.names=T)
|
||||
|
||||
plot.df[,accuracy := accuracy]
|
||||
|
||||
@ -219,15 +88,22 @@ build_plot_dataset <- function(df){
|
||||
|
||||
df <- read_feather(args$infile)
|
||||
plot.df <- build_plot_dataset(df)
|
||||
|
||||
remember(plot.df,args$name)
|
||||
|
||||
|
||||
## df[gmm.ER_pval<0.05]
|
||||
## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
|
||||
## N=factor(N),
|
||||
## m=factor(m))]
|
||||
|
||||
|
||||
## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & !is.na(p.true.in.ci) & (method!="Multiple imputation (Classifier features unobserved)")]
|
||||
## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
|
||||
## p <- p + geom_hline(aes(yintercept=-0.05),linetype=2)
|
||||
|
||||
|
||||
|
||||
## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
|
||||
## print(p)
|
||||
## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
|
||||
|
||||
## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
|
||||
|
||||
@ -10,267 +10,172 @@ parser <- add_argument(parser, "--infile", default="", help="name of the file to
|
||||
parser <- add_argument(parser, "--name", default="", help="The name to safe the data to in the remember file.")
|
||||
args <- parse_args(parser)
|
||||
|
||||
summarize.estimator <- function(df, suffix='naive', coefname='x'){
|
||||
|
||||
part <- df[,c('N',
|
||||
'm',
|
||||
'Bxy',
|
||||
paste0('B',coefname,'y.est.',suffix),
|
||||
paste0('B',coefname,'y.ci.lower.',suffix),
|
||||
paste0('B',coefname,'y.ci.upper.',suffix),
|
||||
'y_explained_variance',
|
||||
'Bzx',
|
||||
'Bzy',
|
||||
'accuracy_imbalance_difference'
|
||||
),
|
||||
with=FALSE]
|
||||
|
||||
true.in.ci <- as.integer((part$Bxy >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (part$Bxy <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]]))
|
||||
zero.in.ci <- as.integer(0 >= part[[paste0('B',coefname,'y.ci.lower.',suffix)]]) & (0 <= part[[paste0('B',coefname,'y.ci.upper.',suffix)]])
|
||||
bias <- part$Bxy - part[[paste0('B',coefname,'y.est.',suffix)]]
|
||||
sign.correct <- as.integer(sign(part$Bxy) == sign(part[[paste0('B',coefname,'y.est.',suffix)]]))
|
||||
|
||||
part <- part[,':='(true.in.ci = true.in.ci,
|
||||
zero.in.ci = zero.in.ci,
|
||||
bias=bias,
|
||||
sign.correct =sign.correct)]
|
||||
|
||||
part.plot <- part[, .(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
|
||||
var.est = var(.SD[[paste0('B',coefname,'y.est.',suffix)]]),
|
||||
est.upper.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.95,na.rm=T),
|
||||
est.lower.95 = quantile(.SD[[paste0('B',coefname,'y.est.',suffix)]],0.05,na.rm=T),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable=coefname,
|
||||
method=suffix
|
||||
),
|
||||
by=c("N","m",'y_explained_variance','Bzx', 'Bzy', 'accuracy_imbalance_difference')
|
||||
]
|
||||
|
||||
return(part.plot)
|
||||
}
|
||||
|
||||
build_plot_dataset <- function(df){
|
||||
x.naive <- df[,.(N, m, Bxy, Bxy.est.naive, Bxy.ci.lower.naive, Bxy.ci.upper.naive)]
|
||||
x.naive <- x.naive[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.naive) & (Bxy <= Bxy.ci.upper.naive)),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.naive) & (0 <= Bxy.ci.upper.naive),
|
||||
bias = Bxy - Bxy.est.naive,
|
||||
Bxy.est.naive = Bxy.est.naive,
|
||||
sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.naive)))]
|
||||
|
||||
x.naive.plot <- x.naive[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bxy.est.naive),
|
||||
var.est = var(Bxy.est.naive),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='x',
|
||||
method='Naive'
|
||||
),
|
||||
by=c('N','m')]
|
||||
x.true <- summarize.estimator(df, 'true','x')
|
||||
|
||||
z.true <- summarize.estimator(df, 'true','z')
|
||||
|
||||
g.naive <- df[,.(N, m, Bgy, Bgy.est.naive, Bgy.ci.lower.naive, Bgy.ci.upper.naive)]
|
||||
g.naive <- g.naive[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.naive) & (Bgy <= Bgy.ci.upper.naive)),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.naive) & (0 <= Bgy.ci.upper.naive),
|
||||
bias = Bgy - Bgy.est.naive,
|
||||
Bgy.est.naive = Bgy.est.naive,
|
||||
sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.naive)))]
|
||||
x.naive <- summarize.estimator(df, 'naive','x')
|
||||
|
||||
g.naive.plot <- g.naive[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bgy.est.naive),
|
||||
var.est = var(Bgy.est.naive),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='g',
|
||||
method='Naive'
|
||||
),
|
||||
by=c('N','m')]
|
||||
z.naive <- summarize.estimator(df,'naive','z')
|
||||
|
||||
x.feasible <- summarize.estimator(df, 'feasible', 'x')
|
||||
|
||||
x.feasible <- df[,.(N, m, Bxy, Bxy.est.feasible, Bxy.ci.lower.feasible, Bxy.ci.upper.feasible)]
|
||||
x.feasible <- x.feasible[,':='(true.in.ci = as.integer((Bxy >= Bxy.ci.lower.feasible) & (Bxy <= Bxy.ci.upper.feasible)),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.feasible) & (0 <= Bxy.ci.upper.feasible),
|
||||
bias = Bxy - Bxy.est.feasible,
|
||||
Bxy.est.feasible = Bxy.est.feasible,
|
||||
sign.correct = as.integer(sign(Bxy) == sign(Bxy.est.feasible)))]
|
||||
z.feasible <- summarize.estimator(df, 'feasible', 'z')
|
||||
|
||||
x.feasible.plot <- x.feasible[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bxy.est.feasible),
|
||||
var.est = var(Bxy.est.feasible),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='x',
|
||||
method='Feasible'
|
||||
),
|
||||
by=c('N','m')]
|
||||
x.amelia.full <- summarize.estimator(df, 'amelia.full', 'x')
|
||||
|
||||
z.amelia.full <- summarize.estimator(df, 'amelia.full', 'z')
|
||||
|
||||
g.feasible <- df[,.(N, m, Bgy, Bgy.est.feasible, Bgy.ci.lower.feasible, Bgy.ci.upper.feasible)]
|
||||
g.feasible <- g.feasible[,':='(true.in.ci = as.integer((Bgy >= Bgy.ci.lower.feasible) & (Bgy <= Bgy.ci.upper.feasible)),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.feasible) & (0 <= Bgy.ci.upper.feasible),
|
||||
bias = Bgy - Bgy.est.feasible,
|
||||
Bgy.est.feasible = Bgy.est.feasible,
|
||||
sign.correct = as.integer(sign(Bgy) == sign(Bgy.est.feasible)))]
|
||||
x.mecor <- summarize.estimator(df, 'mecor', 'x')
|
||||
|
||||
g.feasible.plot <- g.feasible[,.(p.true.in.ci = mean(true.in.ci),
|
||||
mean.bias = mean(bias),
|
||||
mean.est = mean(Bgy.est.feasible),
|
||||
var.est = var(Bgy.est.feasible),
|
||||
N.sims = .N,
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
variable='g',
|
||||
method='Feasible'
|
||||
),
|
||||
by=c('N','m')]
|
||||
z.mecor <- summarize.estimator(df, 'mecor', 'z')
|
||||
|
||||
x.mecor <- summarize.estimator(df, 'mecor', 'x')
|
||||
|
||||
z.mecor <- summarize.estimator(df, 'mecor', 'z')
|
||||
|
||||
x.amelia.full <- df[,.(N, m, Bxy, Bxy.est.true, Bxy.ci.lower.amelia.full, Bxy.ci.upper.amelia.full, Bxy.est.amelia.full)]
|
||||
x.mle <- summarize.estimator(df, 'mle', 'x')
|
||||
|
||||
x.amelia.full <- x.amelia.full[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.full) & (Bxy.est.true <= Bxy.ci.upper.amelia.full),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.amelia.full) & (0 <= Bxy.ci.upper.amelia.full),
|
||||
bias = Bxy.est.true - Bxy.est.amelia.full,
|
||||
sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.full))]
|
||||
z.mle <- summarize.estimator(df, 'mle', 'z')
|
||||
|
||||
x.amelia.full.plot <- x.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bxy.est.amelia.full),
|
||||
var.est = var(Bxy.est.amelia.full),
|
||||
N.sims = .N,
|
||||
variable='x',
|
||||
method='Multiple imputation'
|
||||
),
|
||||
by=c('N','m')]
|
||||
x.zhang <- summarize.estimator(df, 'zhang', 'x')
|
||||
|
||||
z.zhang <- summarize.estimator(df, 'zhang', 'z')
|
||||
|
||||
g.amelia.full <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.full, Bgy.ci.lower.amelia.full, Bgy.ci.upper.amelia.full)]
|
||||
g.amelia.full <- g.amelia.full[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.full) & (Bgy.est.true <= Bgy.ci.upper.amelia.full),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.amelia.full) & (0 <= Bgy.ci.upper.amelia.full),
|
||||
bias = Bgy.est.amelia.full - Bgy.est.true,
|
||||
sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.full))]
|
||||
x.gmm <- summarize.estimator(df, 'gmm', 'x')
|
||||
|
||||
g.amelia.full.plot <- g.amelia.full[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bgy.est.amelia.full),
|
||||
var.est = var(Bgy.est.amelia.full),
|
||||
N.sims = .N,
|
||||
variable='g',
|
||||
method='Multiple imputation'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
## x.amelia.nok <- df[,.(N, m, Bxy.est.true, Bxy.est.amelia.nok, Bxy.ci.lower.amelia.nok, Bxy.ci.upper.amelia.nok)]
|
||||
## x.amelia.nok <- x.amelia.nok[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.amelia.nok) & (Bxy.est.true <= Bxy.ci.upper.amelia.nok),
|
||||
## zero.in.ci = (0 >= Bxy.ci.lower.amelia.nok) & (0 <= Bxy.ci.upper.amelia.nok),
|
||||
## bias = Bxy.est.amelia.nok - Bxy.est.true,
|
||||
## sign.correct = sign(Bxy.est.true) == sign(Bxy.est.amelia.nok))]
|
||||
|
||||
## x.amelia.nok.plot <- x.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
## mean.bias = mean(bias),
|
||||
## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
## mean.est = mean(Bxy.est.amelia.nok),
|
||||
## var.est = var(Bxy.est.amelia.nok),
|
||||
## N.sims = .N,
|
||||
## variable='x',
|
||||
## method='Multiple imputation (Classifier features unobserved)'
|
||||
## ),
|
||||
## by=c('N','m')]
|
||||
|
||||
## g.amelia.nok <- df[,.(N, m, Bgy.est.true, Bgy.est.amelia.nok, Bgy.ci.lower.amelia.nok, Bgy.ci.upper.amelia.nok)]
|
||||
## g.amelia.nok <- g.amelia.nok[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.amelia.nok) & (Bgy.est.true <= Bgy.ci.upper.amelia.nok),
|
||||
## zero.in.ci = (0 >= Bgy.ci.lower.amelia.nok) & (0 <= Bgy.ci.upper.amelia.nok),
|
||||
## bias = Bgy.est.amelia.nok - Bgy.est.true,
|
||||
## sign.correct = sign(Bgy.est.true) == sign(Bgy.est.amelia.nok))]
|
||||
|
||||
## g.amelia.nok.plot <- g.amelia.nok[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
## mean.bias = mean(bias),
|
||||
## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
## mean.est = mean(Bgy.est.amelia.nok),
|
||||
## var.est = var(Bgy.est.amelia.nok),
|
||||
## N.sims = .N,
|
||||
## variable='g',
|
||||
## method="Multiple imputation (Classifier features unobserved)"
|
||||
## ),
|
||||
## by=c('N','m')]
|
||||
|
||||
|
||||
x.mecor <- df[,.(N,m,Bxy.est.true, Bxy.est.mecor,Bxy.lower.mecor, Bxy.upper.mecor)]
|
||||
|
||||
x.mecor <- x.mecor[,':='(true.in.ci = (Bxy.est.true >= Bxy.lower.mecor) & (Bxy.est.true <= Bxy.upper.mecor),
|
||||
zero.in.ci = (0 >= Bxy.lower.mecor) & (0 <= Bxy.upper.mecor),
|
||||
bias = Bxy.est.mecor - Bxy.est.true,
|
||||
sign.correct = sign(Bxy.est.true) == sign(Bxy.est.mecor))]
|
||||
|
||||
x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bxy.est.mecor),
|
||||
var.est = var(Bxy.est.mecor),
|
||||
N.sims = .N,
|
||||
variable='x',
|
||||
method='Regression Calibration'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
g.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.lower.mecor, Bgy.upper.mecor)]
|
||||
|
||||
g.mecor <- g.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.lower.mecor) & (Bgy.est.true <= Bgy.upper.mecor),
|
||||
zero.in.ci = (0 >= Bgy.lower.mecor) & (0 <= Bgy.upper.mecor),
|
||||
bias = Bgy.est.mecor - Bgy.est.true,
|
||||
sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))]
|
||||
|
||||
g.mecor.plot <- g.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bgy.est.mecor),
|
||||
var.est = var(Bgy.est.mecor),
|
||||
N.sims = .N,
|
||||
variable='g',
|
||||
method='Regression Calibration'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
## x.mecor <- df[,.(N,m,Bgy.est.true, Bgy.est.mecor,Bgy.ci.lower.mecor, Bgy.ci.upper.mecor)]
|
||||
|
||||
## x.mecor <- x.mecor[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.mecor) & (Bgy.est.true <= Bgy.ci.upper.mecor),
|
||||
## zero.in.ci = (0 >= Bgy.ci.lower.mecor) & (0 <= Bgy.ci.upper.mecor),
|
||||
## bias = Bgy.est.mecor - Bgy.est.true,
|
||||
## sign.correct = sign(Bgy.est.true) == sign(Bgy.est.mecor))]
|
||||
|
||||
## x.mecor.plot <- x.mecor[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
## mean.bias = mean(bias),
|
||||
## p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
## variable='g',
|
||||
## method='Regression Calibration'
|
||||
## ),
|
||||
## by=c('N','m')]
|
||||
|
||||
|
||||
x.gmm <- df[,.(N,m,Bxy.est.true, Bxy.est.gmm,Bxy.ci.lower.gmm, Bxy.ci.upper.gmm)]
|
||||
x.gmm <- x.gmm[,':='(true.in.ci = (Bxy.est.true >= Bxy.ci.lower.gmm) & (Bxy.est.true <= Bxy.ci.upper.gmm),
|
||||
zero.in.ci = (0 >= Bxy.ci.lower.gmm) & (0 <= Bxy.ci.upper.gmm),
|
||||
bias = Bxy.est.gmm - Bxy.est.true,
|
||||
sign.correct = sign(Bxy.est.true) == sign(Bxy.est.gmm))]
|
||||
|
||||
x.gmm.plot <- x.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bxy.est.gmm),
|
||||
|
||||
var.est = var(Bxy.est.gmm),
|
||||
N.sims = .N,
|
||||
variable='x',
|
||||
method='2SLS+gmm'
|
||||
),
|
||||
by=c('N','m')]
|
||||
|
||||
g.gmm <- df[,.(N,m,Bgy.est.true, Bgy.est.gmm,Bgy.ci.lower.gmm, Bgy.ci.upper.gmm)]
|
||||
g.gmm <- g.gmm[,':='(true.in.ci = (Bgy.est.true >= Bgy.ci.lower.gmm) & (Bgy.est.true <= Bgy.ci.upper.gmm),
|
||||
zero.in.ci = (0 >= Bgy.ci.lower.gmm) & (0 <= Bgy.ci.upper.gmm),
|
||||
bias = Bgy.est.gmm - Bgy.est.true,
|
||||
sign.correct = sign(Bgy.est.true) == sign(Bgy.est.gmm))]
|
||||
|
||||
g.gmm.plot <- g.gmm[,.(p.true.in.ci = mean(as.integer(true.in.ci)),
|
||||
mean.bias = mean(bias),
|
||||
p.sign.correct = mean(as.integer(sign.correct & (! zero.in.ci))),
|
||||
mean.est = mean(Bgy.est.gmm),
|
||||
var.est = var(Bgy.est.gmm),
|
||||
N.sims = .N,
|
||||
variable='g',
|
||||
method='2SLS+gmm'
|
||||
),
|
||||
by=c('N','m')]
|
||||
z.gmm <- summarize.estimator(df, 'gmm', 'z')
|
||||
|
||||
accuracy <- df[,mean(accuracy)]
|
||||
|
||||
plot.df <- rbindlist(list(x.naive.plot,g.naive.plot,x.amelia.full.plot,g.amelia.full.plot,x.mecor.plot, g.mecor.plot, x.gmm.plot, g.gmm.plot, x.feasible.plot, g.feasible.plot),use.names=T)
|
||||
|
||||
plot.df <- rbindlist(list(x.true,z.true,x.naive,z.naive,x.amelia.full,z.amelia.full,x.mecor, z.mecor, x.gmm, z.gmm, x.feasible, z.feasible,z.mle, x.mle, x.zhang, z.zhang, x.gmm, z.gmm),use.names=T)
|
||||
plot.df[,accuracy := accuracy]
|
||||
|
||||
plot.df <- plot.df[,":="(sd.est=sqrt(var.est)/N.sims)]
|
||||
|
||||
return(plot.df)
|
||||
}
|
||||
|
||||
|
||||
df <- read_feather(args$infile)
|
||||
plot.df <- build_plot_dataset(df)
|
||||
plot.df <- read_feather(args$infile)
|
||||
|
||||
# df <- df[apply(df,1,function(x) !any(is.na(x)))]
|
||||
|
||||
if(!('Bzx' %in% names(plot.df)))
|
||||
plot.df[,Bzx:=NA]
|
||||
|
||||
if(!('accuracy_imbalance_difference' %in% names(plot.df)))
|
||||
plot.df[,accuracy_imbalance_difference:=NA]
|
||||
|
||||
unique(plot.df[,'accuracy_imbalance_difference'])
|
||||
|
||||
#plot.df <- build_plot_dataset(df[accuracy_imbalance_difference==0.1][N==700])
|
||||
plot.df <- build_plot_dataset(plot.df)
|
||||
|
||||
remember(plot.df,args$name)
|
||||
|
||||
#ggplot(df,aes(x=Bxy.est.mle)) + geom_histogram() + facet_grid(accuracy_imbalance_difference ~ Bzy)
|
||||
|
||||
## df[gmm.ER_pval<0.05]
|
||||
## ## ## df[gmm.ER_pval<0.05]
|
||||
|
||||
## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
|
||||
## N=factor(N),
|
||||
## m=factor(m))]
|
||||
|
||||
## plot.df.test <- plot.df.test[(variable=='x') & (method!="Multiple imputation (Classifier features unobserved)")]
|
||||
## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
|
||||
## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=0.1),linetype=2)
|
||||
|
||||
## p <- p + geom_pointrange() + facet_grid(N~m,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
|
||||
## print(p)
|
||||
|
||||
## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
|
||||
## N=factor(N),
|
||||
## m=factor(m))]
|
||||
|
||||
## plot.df.test <- plot.df.test[(variable=='z') & (method!="Multiple imputation (Classifier features unobserved)")]
|
||||
## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
|
||||
## p <- p + geom_hline(data=plot.df.test, mapping=aes(yintercept=-0.1),linetype=2)
|
||||
|
||||
## p <- p + geom_pointrange() + facet_grid(m~N,as.table=F,scales='free') + scale_x_discrete(labels=label_wrap_gen(4))
|
||||
## print(p)
|
||||
|
||||
|
||||
## x.mle <- df[,.(N,m,Bxy.est.mle,Bxy.ci.lower.mle, Bxy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
|
||||
## x.mle.plot <- x.mle[,.(mean.est = mean(Bxy.est.mle),
|
||||
## var.est = var(Bxy.est.mle),
|
||||
## N.sims = .N,
|
||||
## variable='z',
|
||||
## method='Bespoke MLE'
|
||||
## ),
|
||||
## by=c("N","m",'y_explained_variance', 'Bzx', 'Bzy','accuracy_imbalance_difference')]
|
||||
|
||||
## z.mle <- df[,.(N,m,Bzy.est.mle,Bzy.ci.lower.mle, Bzy.ci.upper.mle, y_explained_variance, Bzx, Bzy, accuracy_imbalance_difference)]
|
||||
|
||||
## z.mle.plot <- z.mle[,.(mean.est = mean(Bzy.est.mle),
|
||||
## var.est = var(Bzy.est.mle),
|
||||
## N.sims = .N,
|
||||
## variable='z',
|
||||
## method='Bespoke MLE'
|
||||
## ),
|
||||
## by=c("N","m",'y_explained_variance','Bzx')]
|
||||
|
||||
## plot.df <- z.mle.plot
|
||||
## plot.df.test <- plot.df[,':='(method=factor(method,levels=c("Naive","Multiple imputation", "Multiple imputation (Classifier features unobserved)","Regression Calibration","2SLS+gmm","Bespoke MLE", "Feasible"),ordered=T),
|
||||
## N=factor(N),
|
||||
## m=factor(m))]
|
||||
|
||||
## plot.df.test <- plot.df.test[(variable=='z') & (m != 1000) & (m!=500) & (method!="Multiple imputation (Classifier features unobserved)")]
|
||||
## p <- ggplot(plot.df.test, aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method))
|
||||
## p <- p + geom_hline(aes(yintercept=0.2),linetype=2)
|
||||
|
||||
## p <- p + geom_pointrange() + facet_grid(m~Bzx, Bzy,as.table=F) + scale_x_discrete(labels=label_wrap_gen(4))
|
||||
## print(p)
|
||||
|
||||
|
||||
## ## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
|
||||
|
||||
## ggplot(plot.df[variable=='x'], aes(y=mean.est, ymax=mean.est + var.est/2, ymin=mean.est-var.est/2, x=method)) + geom_pointrange() + facet_grid(-m~N) + scale_x_discrete(labels=label_wrap_gen(10))
|
||||
## ## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
|
||||
|
||||
## ggplot(plot.df,aes(y=N,x=m,color=p.sign.correct)) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
|
||||
|
||||
## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
|
||||
## ## ggplot(plot.df,aes(y=N,x=m,color=abs(mean.bias))) + geom_point() + facet_grid(variable ~ method) + scale_color_viridis_c(option='D') + theme_minimal() + xlab("Number of gold standard labels") + ylab("Total sample size")
|
||||
|
||||
@ -6,14 +6,14 @@
|
||||
## Resources
|
||||
#SBATCH --nodes=1
|
||||
## Walltime (12 hours)
|
||||
#SBATCH --time=24:00:00
|
||||
#SBATCH --time=1:00:00
|
||||
## Memory per node
|
||||
#SBATCH --mem=8G
|
||||
#SBATCH --cpus-per-task=1
|
||||
#SBATCH --ntasks-per-node=1
|
||||
#SBATCH --chdir /gscratch/comdata/users/nathante/ml_measurement_error_public/simulations
|
||||
#SBATCH --output=simulation_jobs/%A_%a.out
|
||||
#SBATCH --error=simulation_jobs/%A_%a.out
|
||||
#SBATCH --error=simulation_jobs/%A_%a.err
|
||||
|
||||
TASK_NUM=$(($SLURM_ARRAY_TASK_ID + $1))
|
||||
TASK_CALL=$(sed -n ${TASK_NUM}p $2)
|
||||
|
||||
@ -4,207 +4,324 @@ options(amelia.parallel="no",
|
||||
amelia.ncpus=1)
|
||||
library(Amelia)
|
||||
library(Zelig)
|
||||
library(stats4)
|
||||
library(bbmle)
|
||||
library(matrixStats) # for numerically stable logsumexps
|
||||
|
||||
source("measerr_methods.R") ## for my more generic function.
|
||||
|
||||
## This uses the pseudolikelihood approach from Carroll page 349.
|
||||
## assumes MAR
|
||||
## assumes differential error, but that only depends on Y
|
||||
## inefficient, because pseudolikelihood
|
||||
logistic.correction.pseudo <- function(df){
|
||||
|
||||
## This uses the pseudo-likelihood approach from Carroll page 346.
|
||||
my.pseudo.mle <- function(df){
|
||||
p1.est <- mean(df[w_pred==1]$y.obs==1,na.rm=T)
|
||||
p0.est <- mean(df[w_pred==0]$y.obs==0,na.rm=T)
|
||||
|
||||
nll <- function(B0, Bxy, Bgy){
|
||||
probs <- (1 - p0.est) + (p1.est + p0.est - 1)*plogis(B0 + Bxy * df$x + Bgy * df$g)
|
||||
nll <- function(B0, Bxy, Bzy){
|
||||
|
||||
part1 = sum(log(probs[df$w_pred == 1]))
|
||||
part2 = sum(log(1-probs[df$w_pred == 0]))
|
||||
pw <- vector(mode='numeric',length=nrow(df))
|
||||
dfw1 <- df[w_pred==1]
|
||||
dfw0 <- df[w_pred==0]
|
||||
pw[df$w_pred==1] <- plogis(B0 + Bxy * dfw1$x + Bzy * dfw1$z, log=T)
|
||||
pw[df$w_pred==0] <- plogis(B0 + Bxy * dfw0$x + Bzy * dfw0$z, lower.tail=FALSE, log=T)
|
||||
|
||||
return(-1*(part1 + part2))
|
||||
probs <- colLogSumExps(rbind(log(1 - p0.est), log(p1.est + p0.est - 1) + pw))
|
||||
return(-1*sum(probs))
|
||||
}
|
||||
|
||||
mlefit <- stats4::mle(minuslogl = nll, start = list(B0=0, Bxy=0.0, Bgy=0.0))
|
||||
mlefit <- mle2(minuslogl = nll, start = list(B0=0.0, Bxy=0.0, Bzy=0.0), control=list(maxit=1e6),method='L-BFGS-B')
|
||||
return(mlefit)
|
||||
|
||||
}
|
||||
|
||||
|
||||
## model from Zhang's arxiv paper, with predictions for y
|
||||
## Zhang got this model from Hausman 1998
|
||||
### I think this is actually eqivalent to the pseudo.mle method
|
||||
zhang.mle.iv <- function(df){
|
||||
nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1, ppv=0.9, npv=0.9){
|
||||
df.obs <- df[!is.na(x.obs)]
|
||||
df.unobs <- df[is.na(x.obs)]
|
||||
|
||||
## fpr = 1 - TNR
|
||||
### Problem: accounting for uncertainty in ppv / npv
|
||||
|
||||
ll.w1x1.obs <- with(df.obs[(w_pred==1)], dbinom(x.obs,size=1,prob=ppv,log=T))
|
||||
ll.w0x0.obs <- with(df.obs[(w_pred==0)], dbinom(1-x.obs,size=1,prob=npv,log=T))
|
||||
|
||||
## fnr = 1 - TPR
|
||||
ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
|
||||
ll <- sum(ll.y.obs)
|
||||
ll <- ll + sum(ll.w1x1.obs) + sum(ll.w0x0.obs)
|
||||
|
||||
# unobserved case; integrate out x
|
||||
ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
|
||||
ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
|
||||
|
||||
## case x == 1
|
||||
lls.x.1 <- colLogSumExps(rbind(log(ppv) + ll.x.1, log(1-ppv) + ll.x.0))
|
||||
|
||||
## case x == 0
|
||||
lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
|
||||
|
||||
lls <- colLogSumExps(rbind(lls.x.1, lls.x.0))
|
||||
ll <- ll + sum(lls)
|
||||
return(-ll)
|
||||
}
|
||||
mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf,ppv=0.00001, npv=0.00001),
|
||||
upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf, ppv=0.99999,npv=0.99999),method='L-BFGS-B')
|
||||
return(mlefit)
|
||||
}
|
||||
|
||||
## this is equivalent to the pseudo-liklihood model from Carolla
|
||||
zhang.mle.dv <- function(df){
|
||||
|
||||
nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){
|
||||
df.obs <- df[!is.na(y.obs)]
|
||||
|
||||
## fpr = 1 - TNR
|
||||
ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE))
|
||||
ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE))
|
||||
|
||||
# observed case
|
||||
ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
|
||||
ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
|
||||
ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
|
||||
|
||||
ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1)
|
||||
|
||||
# unobserved case; integrate out y
|
||||
## case y = 1
|
||||
ll.y.1 <- vector(mode='numeric', length=nrow(df))
|
||||
pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
|
||||
## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1)
|
||||
lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1))
|
||||
|
||||
## case y = 0
|
||||
ll.y.0 <- vector(mode='numeric', length=nrow(df))
|
||||
pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
|
||||
|
||||
## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0)
|
||||
lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0))
|
||||
|
||||
lls <- colLogSumExps(rbind(lls.y.1, lls.y.0))
|
||||
ll <- ll + sum(lls)
|
||||
return(-ll)
|
||||
}
|
||||
mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001),
|
||||
upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999))
|
||||
return(mlefit)
|
||||
}
|
||||
|
||||
## This uses the likelihood approach from Carroll page 353.
|
||||
## assumes that we have a good measurement error model
|
||||
logistic.correction.liklihood <- function(df){
|
||||
my.mle <- function(df){
|
||||
|
||||
## liklihood for observed responses
|
||||
nll <- function(B0, Bxy, Bgy, gamma0, gamma_y, gamma_g){
|
||||
nll <- function(B0, Bxy, Bzy, gamma0, gamma_y, gamma_z, gamma_yz){
|
||||
df.obs <- df[!is.na(y.obs)]
|
||||
p.y.obs <- plogis(B0 + Bxy * df.obs$x + Bgy*df.obs$g)
|
||||
p.y.obs[df.obs$y==0] <- 1-p.y.obs[df.obs$y==0]
|
||||
p.s.obs <- plogis(gamma0 + gamma_y * df.obs$y + gamma_g*df.obs$g)
|
||||
p.s.obs[df.obs$w_pred==0] <- 1 - p.s.obs[df.obs$w_pred==0]
|
||||
yobs0 <- df.obs$y==0
|
||||
yobs1 <- df.obs$y==1
|
||||
p.y.obs <- vector(mode='numeric', length=nrow(df.obs))
|
||||
|
||||
p.obs <- p.s.obs * p.y.obs
|
||||
p.y.obs[yobs1] <- plogis(B0 + Bxy * df.obs[yobs1]$x + Bzy*df.obs[yobs1]$z,log=T)
|
||||
p.y.obs[yobs0] <- plogis(B0 + Bxy * df.obs[yobs0]$x + Bzy*df.obs[yobs0]$z,lower.tail=FALSE,log=T)
|
||||
|
||||
wobs0 <- df.obs$w_pred==0
|
||||
wobs1 <- df.obs$w_pred==1
|
||||
p.w.obs <- vector(mode='numeric', length=nrow(df.obs))
|
||||
|
||||
p.w.obs[wobs1] <- plogis(gamma0 + gamma_y * df.obs[wobs1]$y + gamma_z*df.obs[wobs1]$z + df.obs[wobs1]$z*df.obs[wobs1]$y* gamma_yz, log=T)
|
||||
p.w.obs[wobs0] <- plogis(gamma0 + gamma_y * df.obs[wobs0]$y + gamma_z*df.obs[wobs0]$z + df.obs[wobs0]$z*df.obs[wobs0]$y* gamma_yz, lower.tail=FALSE, log=T)
|
||||
|
||||
p.obs <- p.w.obs + p.y.obs
|
||||
|
||||
df.unobs <- df[is.na(y.obs)]
|
||||
|
||||
p.unobs.1 <- plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g)*plogis(gamma0 + gamma_y + gamma_g*df.unobs$g)
|
||||
p.unobs.0 <- (1-plogis(B0 + Bxy * df.unobs$x + Bgy*df.unobs$g))*plogis(gamma0 + gamma_g*df.unobs$g)
|
||||
p.unobs <- p.unobs.1 + p.unobs.0
|
||||
p.unobs[df.unobs$w_pred==0] <- 1 - p.unobs[df.unobs$w_pred==0]
|
||||
p.unobs.0 <- vector(mode='numeric',length=nrow(df.unobs))
|
||||
p.unobs.1 <- vector(mode='numeric',length=nrow(df.unobs))
|
||||
|
||||
wunobs.0 <- df.unobs$w_pred == 0
|
||||
wunobs.1 <- df.unobs$w_pred == 1
|
||||
|
||||
p.unobs.0[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.1]$z + df.unobs[wunobs.1]$z*gamma_yz, log=T)
|
||||
|
||||
p.unobs.0[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T) + plogis(gamma0 + gamma_y + gamma_z*df.unobs[wunobs.0]$z + df.unobs[wunobs.0]$z*gamma_yz, lower.tail=FALSE, log=T)
|
||||
|
||||
p.unobs.1[wunobs.1] <- plogis(B0 + Bxy * df.unobs[wunobs.1]$x + Bzy*df.unobs[wunobs.1]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.1]$z, log=T)
|
||||
|
||||
p.unobs.1[wunobs.0] <- plogis(B0 + Bxy * df.unobs[wunobs.0]$x + Bzy*df.unobs[wunobs.0]$z, log=T, lower.tail=FALSE) + plogis(gamma0 + gamma_z*df.unobs[wunobs.0]$z, lower.tail=FALSE, log=T)
|
||||
|
||||
p.unobs <- colLogSumExps(rbind(p.unobs.1, p.unobs.0))
|
||||
|
||||
p <- c(p.obs, p.unobs)
|
||||
|
||||
return(-1*(sum(log(p))))
|
||||
return(-1*(sum(p)))
|
||||
}
|
||||
|
||||
mlefit <- stats4::mle(minuslogl = nll, start = list(B0=1, Bxy=0,Bgy=0, gamma0=5, gamma_y=0, gamma_g=0))
|
||||
mlefit <- mle2(minuslogl = nll, start = list(B0=0, Bxy=0,Bzy=0, gamma0=0, gamma_y=0, gamma_z=0, gamma_yz=0), control=list(maxit=1e6),method='L-BFGS-B')
|
||||
|
||||
return(mlefit)
|
||||
}
|
||||
|
||||
|
||||
logistic <- function(x) {1/(1+exp(-1*x))}
|
||||
|
||||
run_simulation_depvar <- function(df, result){
|
||||
run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~y){
|
||||
|
||||
accuracy <- df[,mean(w_pred==y)]
|
||||
result <- append(result, list(accuracy=accuracy))
|
||||
|
||||
(model.true <- glm(y ~ x + g, data=df,family=binomial(link='logit')))
|
||||
(model.true <- glm(y ~ x + z, data=df,family=binomial(link='logit')))
|
||||
true.ci.Bxy <- confint(model.true)['x',]
|
||||
true.ci.Bgy <- confint(model.true)['g',]
|
||||
true.ci.Bzy <- confint(model.true)['z',]
|
||||
|
||||
result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
|
||||
Bgy.est.true=coef(model.true)['g'],
|
||||
Bzy.est.true=coef(model.true)['z'],
|
||||
Bxy.ci.upper.true = true.ci.Bxy[2],
|
||||
Bxy.ci.lower.true = true.ci.Bxy[1],
|
||||
Bgy.ci.upper.true = true.ci.Bgy[2],
|
||||
Bgy.ci.lower.true = true.ci.Bgy[1]))
|
||||
Bzy.ci.upper.true = true.ci.Bzy[2],
|
||||
Bzy.ci.lower.true = true.ci.Bzy[1]))
|
||||
|
||||
(model.feasible <- glm(y.obs~x+g,data=df,family=binomial(link='logit')))
|
||||
(model.feasible <- glm(y.obs~x+z,data=df,family=binomial(link='logit')))
|
||||
|
||||
feasible.ci.Bxy <- confint(model.feasible)['x',]
|
||||
result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x'],
|
||||
Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
|
||||
Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
|
||||
|
||||
feasible.ci.Bgy <- confint(model.feasible)['g',]
|
||||
result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'],
|
||||
Bgy.ci.upper.feasible = feasible.ci.Bgy[2],
|
||||
Bgy.ci.lower.feasible = feasible.ci.Bgy[1]))
|
||||
feasible.ci.Bzy <- confint(model.feasible)['z',]
|
||||
result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'],
|
||||
Bzy.ci.upper.feasible = feasible.ci.Bzy[2],
|
||||
Bzy.ci.lower.feasible = feasible.ci.Bzy[1]))
|
||||
|
||||
(model.naive <- glm(w_pred~x+g, data=df, family=binomial(link='logit')))
|
||||
(model.naive <- glm(w_pred~x+z, data=df, family=binomial(link='logit')))
|
||||
|
||||
naive.ci.Bxy <- confint(model.naive)['x',]
|
||||
naive.ci.Bgy <- confint(model.naive)['g',]
|
||||
naive.ci.Bzy <- confint(model.naive)['z',]
|
||||
|
||||
result <- append(result, list(Bxy.est.naive=coef(model.naive)['x'],
|
||||
Bgy.est.naive=coef(model.naive)['g'],
|
||||
Bzy.est.naive=coef(model.naive)['z'],
|
||||
Bxy.ci.upper.naive = naive.ci.Bxy[2],
|
||||
Bxy.ci.lower.naive = naive.ci.Bxy[1],
|
||||
Bgy.ci.upper.naive = naive.ci.Bgy[2],
|
||||
Bgy.ci.lower.naive = naive.ci.Bgy[1]))
|
||||
Bzy.ci.upper.naive = naive.ci.Bzy[2],
|
||||
Bzy.ci.lower.naive = naive.ci.Bzy[1]))
|
||||
|
||||
|
||||
(model.naive.cont <- lm(w~x+g, data=df))
|
||||
(model.naive.cont <- lm(w~x+z, data=df))
|
||||
naivecont.ci.Bxy <- confint(model.naive.cont)['x',]
|
||||
naivecont.ci.Bgy <- confint(model.naive.cont)['g',]
|
||||
naivecont.ci.Bzy <- confint(model.naive.cont)['z',]
|
||||
|
||||
## my implementatoin of liklihood based correction
|
||||
mod.caroll.lik <- logistic.correction.liklihood(df)
|
||||
coef <- coef(mod.caroll.lik)
|
||||
ci <- confint(mod.caroll.lik)
|
||||
|
||||
temp.df <- copy(df)
|
||||
temp.df[,y:=y.obs]
|
||||
mod.caroll.lik <- measerr_mle_dv(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula)
|
||||
fisher.info <- solve(mod.caroll.lik$hessian)
|
||||
coef <- mod.caroll.lik$par
|
||||
ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
|
||||
ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
|
||||
result <- append(result,
|
||||
list(Bxy.est.mle = coef['Bxy'],
|
||||
Bxy.ci.upper.mle = ci['Bxy','97.5 %'],
|
||||
Bxy.ci.lower.mle = ci['Bxy','2.5 %'],
|
||||
Bgy.est.mle = coef['Bgy'],
|
||||
Bgy.ci.upper.mle = ci['Bgy','97.5 %'],
|
||||
Bgy.ci.lower.mle = ci['Bgy','2.5 %']))
|
||||
list(Bxy.est.mle = coef['x'],
|
||||
Bxy.ci.upper.mle = ci.upper['x'],
|
||||
Bxy.ci.lower.mle = ci.lower['x'],
|
||||
Bzy.est.mle = coef['z'],
|
||||
Bzy.ci.upper.mle = ci.upper['z'],
|
||||
Bzy.ci.lower.mle = ci.lower['z']))
|
||||
|
||||
|
||||
## my implementatoin of liklihood based correction
|
||||
mod.caroll.pseudo <- logistic.correction.pseudo(df)
|
||||
coef <- coef(mod.caroll.pseudo)
|
||||
ci <- confint(mod.caroll.pseudo)
|
||||
mod.zhang <- zhang.mle.dv(df)
|
||||
coef <- coef(mod.zhang)
|
||||
ci <- confint(mod.zhang,method='quad')
|
||||
|
||||
result <- append(result,
|
||||
list(Bxy.est.pseudo = coef['Bxy'],
|
||||
Bxy.ci.upper.pseudo = ci['Bxy','97.5 %'],
|
||||
Bxy.ci.lower.pseudo = ci['Bxy','2.5 %'],
|
||||
Bgy.est.pseudo = coef['Bgy'],
|
||||
Bgy.ci.upper.pseudo = ci['Bgy','97.5 %'],
|
||||
Bgy.ci.lower.pseudo = ci['Bgy','2.5 %']))
|
||||
list(Bxy.est.zhang = coef['Bxy'],
|
||||
Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
|
||||
Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
|
||||
Bzy.est.zhang = coef['Bzy'],
|
||||
Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
|
||||
Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
|
||||
|
||||
|
||||
# amelia says use normal distribution for binary variables.
|
||||
amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w_pred'))
|
||||
mod.amelia.k <- zelig(y.obs~x+g, model='ls', data=amelia.out.k$imputations, cite=FALSE)
|
||||
(coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
|
||||
tryCatch({
|
||||
amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('y','ystar','w'))
|
||||
mod.amelia.k <- zelig(y.obs~x+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
|
||||
(coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
|
||||
est.x.mi <- coefse['x','Estimate']
|
||||
est.x.se <- coefse['x','Std.Error']
|
||||
result <- append(result,
|
||||
list(Bxy.est.amelia.full = est.x.mi,
|
||||
Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
|
||||
Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
|
||||
))
|
||||
|
||||
est.x.mi <- coefse['x','Estimate']
|
||||
est.x.se <- coefse['x','Std.Error']
|
||||
result <- append(result,
|
||||
list(Bxy.est.amelia.full = est.x.mi,
|
||||
Bxy.ci.upper.amelia.full = est.x.mi + 1.96 * est.x.se,
|
||||
Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
|
||||
))
|
||||
est.z.mi <- coefse['z','Estimate']
|
||||
est.z.se <- coefse['z','Std.Error']
|
||||
|
||||
est.g.mi <- coefse['g','Estimate']
|
||||
est.g.se <- coefse['g','Std.Error']
|
||||
result <- append(result,
|
||||
list(Bzy.est.amelia.full = est.z.mi,
|
||||
Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
|
||||
Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
|
||||
))
|
||||
|
||||
},
|
||||
error = function(e){
|
||||
message("An error occurred:\n",e)
|
||||
result$error <- paste0(result$error,'\n', e)
|
||||
})
|
||||
|
||||
result <- append(result,
|
||||
list(Bgy.est.amelia.full = est.g.mi,
|
||||
Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se,
|
||||
Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se
|
||||
))
|
||||
|
||||
return(result)
|
||||
|
||||
}
|
||||
|
||||
run_simulation <- function(df, result){
|
||||
|
||||
## outcome_formula, proxy_formula, and truth_formula are passed to measerr_mle
|
||||
run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x, truth_formula=x~z){
|
||||
|
||||
accuracy <- df[,mean(w_pred==x)]
|
||||
result <- append(result, list(accuracy=accuracy))
|
||||
|
||||
(model.true <- lm(y ~ x + g, data=df))
|
||||
(model.true <- lm(y ~ x + z, data=df))
|
||||
true.ci.Bxy <- confint(model.true)['x',]
|
||||
true.ci.Bgy <- confint(model.true)['g',]
|
||||
true.ci.Bzy <- confint(model.true)['z',]
|
||||
|
||||
result <- append(result, list(Bxy.est.true=coef(model.true)['x'],
|
||||
Bgy.est.true=coef(model.true)['g'],
|
||||
Bzy.est.true=coef(model.true)['z'],
|
||||
Bxy.ci.upper.true = true.ci.Bxy[2],
|
||||
Bxy.ci.lower.true = true.ci.Bxy[1],
|
||||
Bgy.ci.upper.true = true.ci.Bgy[2],
|
||||
Bgy.ci.lower.true = true.ci.Bgy[1]))
|
||||
Bzy.ci.upper.true = true.ci.Bzy[2],
|
||||
Bzy.ci.lower.true = true.ci.Bzy[1]))
|
||||
|
||||
(model.feasible <- lm(y~x.obs+g,data=df))
|
||||
(model.feasible <- lm(y~x.obs+z,data=df))
|
||||
|
||||
feasible.ci.Bxy <- confint(model.feasible)['x.obs',]
|
||||
result <- append(result, list(Bxy.est.feasible=coef(model.feasible)['x.obs'],
|
||||
Bxy.ci.upper.feasible = feasible.ci.Bxy[2],
|
||||
Bxy.ci.lower.feasible = feasible.ci.Bxy[1]))
|
||||
|
||||
feasible.ci.Bgy <- confint(model.feasible)['g',]
|
||||
result <- append(result, list(Bgy.est.feasible=coef(model.feasible)['g'],
|
||||
Bgy.ci.upper.feasible = feasible.ci.Bgy[2],
|
||||
Bgy.ci.lower.feasible = feasible.ci.Bgy[1]))
|
||||
feasible.ci.Bzy <- confint(model.feasible)['z',]
|
||||
result <- append(result, list(Bzy.est.feasible=coef(model.feasible)['z'],
|
||||
Bzy.ci.upper.feasible = feasible.ci.Bzy[2],
|
||||
Bzy.ci.lower.feasible = feasible.ci.Bzy[1]))
|
||||
|
||||
(model.naive <- lm(y~w+g, data=df))
|
||||
(model.naive <- lm(y~w_pred+z, data=df))
|
||||
|
||||
naive.ci.Bxy <- confint(model.naive)['w',]
|
||||
naive.ci.Bgy <- confint(model.naive)['g',]
|
||||
naive.ci.Bxy <- confint(model.naive)['w_pred',]
|
||||
naive.ci.Bzy <- confint(model.naive)['z',]
|
||||
|
||||
result <- append(result, list(Bxy.est.naive=coef(model.naive)['w'],
|
||||
Bgy.est.naive=coef(model.naive)['g'],
|
||||
result <- append(result, list(Bxy.est.naive=coef(model.naive)['w_pred'],
|
||||
Bzy.est.naive=coef(model.naive)['z'],
|
||||
Bxy.ci.upper.naive = naive.ci.Bxy[2],
|
||||
Bxy.ci.lower.naive = naive.ci.Bxy[1],
|
||||
Bgy.ci.upper.naive = naive.ci.Bgy[2],
|
||||
Bgy.ci.lower.naive = naive.ci.Bgy[1]))
|
||||
Bzy.ci.upper.naive = naive.ci.Bzy[2],
|
||||
Bzy.ci.lower.naive = naive.ci.Bzy[1]))
|
||||
|
||||
|
||||
tryCatch({
|
||||
amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred'))
|
||||
mod.amelia.k <- zelig(y~x.obs+g, model='ls', data=amelia.out.k$imputations, cite=FALSE)
|
||||
mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
|
||||
(coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
|
||||
|
||||
est.x.mi <- coefse['x.obs','Estimate']
|
||||
@ -215,15 +332,65 @@ run_simulation <- function(df, result){
|
||||
Bxy.ci.lower.amelia.full = est.x.mi - 1.96 * est.x.se
|
||||
))
|
||||
|
||||
est.g.mi <- coefse['g','Estimate']
|
||||
est.g.se <- coefse['g','Std.Error']
|
||||
est.z.mi <- coefse['z','Estimate']
|
||||
est.z.se <- coefse['z','Std.Error']
|
||||
|
||||
result <- append(result,
|
||||
list(Bgy.est.amelia.full = est.g.mi,
|
||||
Bgy.ci.upper.amelia.full = est.g.mi + 1.96 * est.g.se,
|
||||
Bgy.ci.lower.amelia.full = est.g.mi - 1.96 * est.g.se
|
||||
list(Bzy.est.amelia.full = est.z.mi,
|
||||
Bzy.ci.upper.amelia.full = est.z.mi + 1.96 * est.z.se,
|
||||
Bzy.ci.lower.amelia.full = est.z.mi - 1.96 * est.z.se
|
||||
))
|
||||
|
||||
},
|
||||
error = function(e){
|
||||
message("An error occurred:\n",e)
|
||||
result$error <-paste0(result$error,'\n', e)
|
||||
}
|
||||
)
|
||||
|
||||
tryCatch({
|
||||
temp.df <- copy(df)
|
||||
temp.df <- temp.df[,x:=x.obs]
|
||||
mod.caroll.lik <- measerr_mle(temp.df, outcome_formula=outcome_formula, proxy_formula=proxy_formula, truth_formula=truth_formula)
|
||||
fisher.info <- solve(mod.caroll.lik$hessian)
|
||||
coef <- mod.caroll.lik$par
|
||||
ci.upper <- coef + sqrt(diag(fisher.info)) * 1.96
|
||||
ci.lower <- coef - sqrt(diag(fisher.info)) * 1.96
|
||||
|
||||
|
||||
result <- append(result,
|
||||
list(Bxy.est.mle = coef['x'],
|
||||
Bxy.ci.upper.mle = ci.upper['x'],
|
||||
Bxy.ci.lower.mle = ci.lower['x'],
|
||||
Bzy.est.mle = coef['z'],
|
||||
Bzy.ci.upper.mle = ci.upper['z'],
|
||||
Bzy.ci.lower.mle = ci.lower['z']))
|
||||
},
|
||||
|
||||
error = function(e){
|
||||
message("An error occurred:\n",e)
|
||||
result$error <- paste0(result$error,'\n', e)
|
||||
})
|
||||
|
||||
tryCatch({
|
||||
|
||||
mod.zhang.lik <- zhang.mle.iv(df)
|
||||
coef <- coef(mod.zhang.lik)
|
||||
ci <- confint(mod.zhang.lik,method='quad')
|
||||
result <- append(result,
|
||||
list(Bxy.est.zhang = coef['Bxy'],
|
||||
Bxy.ci.upper.zhang = ci['Bxy','97.5 %'],
|
||||
Bxy.ci.lower.zhang = ci['Bxy','2.5 %'],
|
||||
Bzy.est.zhang = coef['Bzy'],
|
||||
Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
|
||||
Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
|
||||
},
|
||||
|
||||
error = function(e){
|
||||
message("An error occurred:\n",e)
|
||||
result$error <- paste0(result$error,'\n', e)
|
||||
})
|
||||
|
||||
## What if we can't observe k -- most realistic scenario. We can't include all the ML features in a model.
|
||||
## amelia.out.nok <- amelia(df, m=200, p2s=0, idvars=c("x","w_pred"), noms=noms)
|
||||
## mod.amelia.nok <- zelig(y~x.obs+g, model='ls', data=amelia.out.nok$imputations, cite=FALSE)
|
||||
@ -255,10 +422,10 @@ run_simulation <- function(df, result){
|
||||
df <- df[order(x.obs)]
|
||||
y <- df[,y]
|
||||
x <- df[,x.obs]
|
||||
g <- df[,g]
|
||||
w <- df[,w]
|
||||
z <- df[,z]
|
||||
w <- df[,w_pred]
|
||||
# gmm gets pretty close
|
||||
(gmm.res <- predicted_covariates(y, x, g, w, v, train, p, max_iter=100, verbose=TRUE))
|
||||
(gmm.res <- predicted_covariates(y, x, z, w, v, train, p, max_iter=100, verbose=TRUE))
|
||||
|
||||
result <- append(result,
|
||||
list(Bxy.est.gmm = gmm.res$beta[1,1],
|
||||
@ -268,28 +435,34 @@ run_simulation <- function(df, result){
|
||||
))
|
||||
|
||||
result <- append(result,
|
||||
list(Bgy.est.gmm = gmm.res$beta[2,1],
|
||||
Bgy.ci.upper.gmm = gmm.res$confint[2,2],
|
||||
Bgy.ci.lower.gmm = gmm.res$confint[2,1]))
|
||||
list(Bzy.est.gmm = gmm.res$beta[2,1],
|
||||
Bzy.ci.upper.gmm = gmm.res$confint[2,2],
|
||||
Bzy.ci.lower.gmm = gmm.res$confint[2,1]))
|
||||
|
||||
|
||||
mod.calibrated.mle <- mecor(y ~ MeasError(w, reference = x.obs) + g, df, B=400, method='efficient')
|
||||
tryCatch({
|
||||
mod.calibrated.mle <- mecor(y ~ MeasError(w_pred, reference = x.obs) + z, df, B=400, method='efficient')
|
||||
(mod.calibrated.mle)
|
||||
(mecor.ci <- summary(mod.calibrated.mle)$c$ci['x.obs',])
|
||||
result <- append(result, list(
|
||||
Bxy.est.mecor = mecor.ci['Estimate'],
|
||||
Bxy.upper.mecor = mecor.ci['UCI'],
|
||||
Bxy.lower.mecor = mecor.ci['LCI'])
|
||||
Bxy.ci.upper.mecor = mecor.ci['UCI'],
|
||||
Bxy.ci.lower.mecor = mecor.ci['LCI'])
|
||||
)
|
||||
|
||||
(mecor.ci <- summary(mod.calibrated.mle)$c$ci['g',])
|
||||
(mecor.ci <- summary(mod.calibrated.mle)$c$ci['z',])
|
||||
|
||||
result <- append(result, list(
|
||||
Bgy.est.mecor = mecor.ci['Estimate'],
|
||||
Bgy.upper.mecor = mecor.ci['UCI'],
|
||||
Bgy.lower.mecor = mecor.ci['LCI'])
|
||||
Bzy.est.mecor = mecor.ci['Estimate'],
|
||||
Bzy.ci.upper.mecor = mecor.ci['UCI'],
|
||||
Bzy.ci.lower.mecor = mecor.ci['LCI'])
|
||||
)
|
||||
|
||||
},
|
||||
error = function(e){
|
||||
message("An error occurred:\n",e)
|
||||
result$error <- paste0(result$error, '\n', e)
|
||||
}
|
||||
)
|
||||
## clean up memory
|
||||
## rm(list=c("df","y","x","g","w","v","train","p","amelia.out.k","amelia.out.nok", "mod.calibrated.mle","gmm.res","mod.amelia.k","mod.amelia.nok", "model.true","model.naive","model.feasible"))
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user