Update the core 4 simulations.
This commit is contained in:
parent
2cd447c327
commit
47e9367ed5
@ -71,21 +71,27 @@ parser <- add_argument(parser, "--outfile", help='output file', default='example
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.05)
|
||||
# parser <- add_argument(parser, "--zx_explained_variance", help='what proportion of the variance of x can be explained by z?', default=0.3)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
|
||||
parser <- add_argument(parser, "--Bzx", help='coefficient of z on x?', default=1)
|
||||
args <- parse_args(parser)
|
||||
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
|
||||
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~x")
|
||||
|
||||
parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
|
||||
parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
|
||||
parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
|
||||
|
||||
args <- parse_args(parser)
|
||||
B0 <- 0
|
||||
Bxy <- 0.3
|
||||
Bzy <- -0.3
|
||||
Bxy <- args$Bxy
|
||||
Bzy <- args$Bzy
|
||||
Bzx <- args$Bzx
|
||||
|
||||
if (args$m < args$N){
|
||||
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, Bzx, seed=args$seed + 500, y_explained_variance = args$y_explained_variance, prediction_accuracy=args$prediction_accuracy)
|
||||
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'Bzx'=Bzx, 'seed'=args$seed, 'y_explained_variance' = args$y_explained_variance, 'zx_explained_variance' = args$zx_explained_variance, "prediction_accuracy"=args$prediction_accuracy, "error"="")
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'outcome_formula'=args$outcome_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, error='')
|
||||
|
||||
outline <- run_simulation(df, result)
|
||||
outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula))
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
|
||||
@ -31,11 +31,11 @@ source("simulation_base.R")
|
||||
|
||||
## one way to do it is by adding correlation to x.obs and y that isn't in w.
|
||||
## in other words, the model is missing an important feature of x.obs that's related to y.
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, y_bias=-0.8){
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.025, prediction_accuracy=0.73, y_bias=-0.8,accuracy_imbalance_difference=0.3){
|
||||
set.seed(seed)
|
||||
# make w and y dependent
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, Bzx * z + 0.5)
|
||||
z <- rbinom(N, 1, plogis(qlogis(0.5)))
|
||||
x <- rbinom(N, 1, plogis(Bzx * z + qlogis(0.5)))
|
||||
|
||||
y.var.epsilon <- (var(Bzy * z) + var(Bxy *x) + 2*cov(Bzy*z,Bxy*x)) * ((1-y_explained_variance)/y_explained_variance)
|
||||
y.epsilon <- rnorm(N, sd = sqrt(y.var.epsilon))
|
||||
@ -50,38 +50,94 @@ simulate_data <- function(N, m, B0, Bxy, Bzx, Bzy, seed, y_explained_variance=0.
|
||||
}
|
||||
|
||||
## probablity of an error is correlated with y
|
||||
p.correct <- plogis(y_bias*scale(y) + qlogis(prediction_accuracy))
|
||||
## pz <- mean(z)
|
||||
## accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
|
||||
|
||||
acc.x0 <- p.correct[df[,x==0]]
|
||||
acc.x1 <- p.correct[df[,x==1]]
|
||||
## # this works because of conditional probability
|
||||
## accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
|
||||
## accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
|
||||
|
||||
df[x==0,w:=rlogis(.N,qlogis(1-acc.x0))]
|
||||
df[x==1,w:=rlogis(.N,qlogis(acc.x1))]
|
||||
## z0x0 <- df[(z==0) & (x==0)]$x
|
||||
## z0x1 <- df[(z==0) & (x==1)]$x
|
||||
## z1x0 <- df[(z==1) & (x==0)]$x
|
||||
## z1x1 <- df[(z==1) & (x==1)]$x
|
||||
|
||||
df[,w_pred := as.integer(w>0.5)]
|
||||
## yz0x0 <- df[(z==0) & (x==0)]$y
|
||||
## yz0x1 <- df[(z==0) & (x==1)]$y
|
||||
## yz1x0 <- df[(z==1) & (x==0)]$y
|
||||
## yz1x1 <- df[(z==1) & (x==1)]$y
|
||||
|
||||
## nz0x0 <- nrow(df[(z==0) & (x==0)])
|
||||
## nz0x1 <- nrow(df[(z==0) & (x==1)])
|
||||
## nz1x0 <- nrow(df[(z==1) & (x==0)])
|
||||
## nz1x1 <- nrow(df[(z==1) & (x==1)])
|
||||
|
||||
## yz1 <- df[z==1]$y
|
||||
## yz1 <- df[z==1]$y
|
||||
|
||||
## # tranform yz0.1 into a logistic distribution with mean accuracy_z0
|
||||
## acc.z0x0 <- plogis(0.5*scale(yz0x0) + qlogis(accuracy_z0))
|
||||
## acc.z0x1 <- plogis(0.5*scale(yz0x1) + qlogis(accuracy_z0))
|
||||
## acc.z1x0 <- plogis(1.5*scale(yz1x0) + qlogis(accuracy_z1))
|
||||
## acc.z1x1 <- plogis(1.5*scale(yz1x1) + qlogis(accuracy_z1))
|
||||
|
||||
## w0z0x0 <- (1-z0x0)**2 + (-1)**(1-z0x0) * acc.z0x0
|
||||
## w0z0x1 <- (1-z0x1)**2 + (-1)**(1-z0x1) * acc.z0x1
|
||||
## w0z1x0 <- (1-z1x0)**2 + (-1)**(1-z1x0) * acc.z1x0
|
||||
## w0z1x1 <- (1-z1x1)**2 + (-1)**(1-z1x1) * acc.z1x1
|
||||
|
||||
## ##perrorz0 <- w0z0*(pyz0)
|
||||
## ##perrorz1 <- w0z1*(pyz1)
|
||||
|
||||
## w0z0x0.noisy.odds <- rlogis(nz0x0,qlogis(w0z0x0))
|
||||
## w0z0x1.noisy.odds <- rlogis(nz0x1,qlogis(w0z0x1))
|
||||
## w0z1x0.noisy.odds <- rlogis(nz1x0,qlogis(w0z1x0))
|
||||
## w0z1x1.noisy.odds <- rlogis(nz1x1,qlogis(w0z1x1))
|
||||
|
||||
## df[(z==0)&(x==0),w:=plogis(w0z0x0.noisy.odds)]
|
||||
## df[(z==0)&(x==1),w:=plogis(w0z0x1.noisy.odds)]
|
||||
## df[(z==1)&(x==0),w:=plogis(w0z1x0.noisy.odds)]
|
||||
## df[(z==1)&(x==1),w:=plogis(w0z1x1.noisy.odds)]
|
||||
|
||||
## df[,w_pred:=as.integer(w > 0.5)]
|
||||
## print(mean(df[z==0]$x == df[z==0]$w_pred))
|
||||
## print(mean(df[z==1]$x == df[z==1]$w_pred))
|
||||
## print(mean(df$w_pred == df$x))
|
||||
|
||||
odds.x1 <- qlogis(prediction_accuracy) + y_bias*qlogis(pnorm(scale(df[x==1]$y)))
|
||||
odds.x0 <- qlogis(prediction_accuracy,lower.tail=F) + y_bias*qlogis(pnorm(scale(df[x==0]$y)))
|
||||
|
||||
## acc.x0 <- p.correct[df[,x==0]]
|
||||
## acc.x1 <- p.correct[df[,x==1]]
|
||||
|
||||
df[x==0,w:=plogis(rlogis(.N,odds.x0))]
|
||||
df[x==1,w:=plogis(rlogis(.N,odds.x1))]
|
||||
|
||||
df[,w_pred := as.integer(w > 0.5)]
|
||||
|
||||
print(mean(df[z==0]$x == df[z==0]$w_pred))
|
||||
print(mean(df[z==1]$x == df[z==1]$w_pred))
|
||||
print(mean(df$w_pred == df$x))
|
||||
print(mean(df[y>=0]$w_pred == df[y>=0]$x))
|
||||
print(mean(df[y<=0]$w_pred == df[y<=0]$x))
|
||||
|
||||
return(df)
|
||||
}
|
||||
|
||||
parser <- arg_parser("Simulate data and fit corrected models")
|
||||
parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
|
||||
aparser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=51, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.01)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.1)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.8)
|
||||
parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzx", help='Effect of z on x', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzy", help='Effect of z on y', default=-0.3)
|
||||
parser <- add_argument(parser, "--Bxy", help='Effect of z on y', default=0.3)
|
||||
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~x*y")
|
||||
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
|
||||
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*z*x")
|
||||
parser <- add_argument(parser, "--y_bias", help='coefficient of y on the probability a classification is correct', default=-0.75)
|
||||
parser <- add_argument(parser, "--truth_formula", help='formula for the true variable', default="x~z")
|
||||
|
||||
args <- parse_args(parser)
|
||||
|
||||
@ -94,16 +150,17 @@ if(args$m < args$N){
|
||||
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzx, Bzy, args$seed, args$y_explained_variance, args$prediction_accuracy, y_bias=args$y_bias)
|
||||
|
||||
## df.pc <- df[,.(x,y,z,w_pred)]
|
||||
## df.pc <- df[,.(x,y,z,w_pred,w)]
|
||||
## # df.pc <- df.pc[,err:=x-w_pred]
|
||||
## pc.df <- pc(suffStat=list(C=cor(df.pc),n=nrow(df.pc)),indepTest=gaussCItest,labels=names(df.pc),alpha=0.05)
|
||||
## plot(pc.df)
|
||||
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'y_bias'=args$y_bias,error='')
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy, Bzx=args$Bzx, 'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference, 'y_bias'=args$y_bias,'outcome_formula'=args$outcome_formula, 'proxy_formula'=args$proxy_formula,truth_formula=args$truth_formula, error='')
|
||||
|
||||
outline <- run_simulation(df, result, outcome_formula=y~x+z, proxy_formula=as.formula(args$proxy_formula), truth_formula=x~z)
|
||||
outline <- run_simulation(df, result, outcome_formula=as.formula(args$outcome_formula), proxy_formula=as.formula(args$proxy_formula), truth_formula=as.formula(args$truth_formula))
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline), fill=TRUE)
|
||||
|
||||
109
simulations/03_depvar.R
Normal file
109
simulations/03_depvar.R
Normal file
@ -0,0 +1,109 @@
|
||||
### EXAMPLE 1: demonstrates how measurement error can lead to a type sign error in a covariate
|
||||
### What kind of data invalidates fong + tyler?
|
||||
### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
|
||||
### Even when you include the proxy variable in the regression.
|
||||
### But with some ground truth and multiple imputation, you can fix it.
|
||||
|
||||
library(argparser)
|
||||
library(mecor)
|
||||
library(ggplot2)
|
||||
library(data.table)
|
||||
library(filelock)
|
||||
library(arrow)
|
||||
library(Amelia)
|
||||
library(Zelig)
|
||||
library(predictionError)
|
||||
options(amelia.parallel="no",
|
||||
amelia.ncpus=1)
|
||||
setDTthreads(40)
|
||||
|
||||
source("simulation_base.R")
|
||||
|
||||
## SETUP:
|
||||
### we want to estimate x -> y; x is MAR
|
||||
### we have x -> k; k -> w; x -> w is used to predict x via the model w.
|
||||
### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
|
||||
### The labels x are binary, but the model provides a continuous predictor
|
||||
|
||||
### simulation:
|
||||
#### how much power do we get from the model in the first place? (sweeping N and m)
|
||||
####
|
||||
|
||||
## one way to do it is by adding correlation to x.obs and y that isn't in w.
|
||||
## in other words, the model is missing an important feature of x.obs that's related to y.
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73){
|
||||
set.seed(seed)
|
||||
|
||||
# make w and y dependent
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, 0.5)
|
||||
|
||||
ystar <- Bzy * z + Bxy * x + B0
|
||||
y <- rbinom(N,1,plogis(ystar))
|
||||
|
||||
# glm(y ~ x + z, family="binomial")
|
||||
|
||||
df <- data.table(x=x,y=y,ystar=ystar,z=z)
|
||||
|
||||
if(m < N){
|
||||
df <- df[sample(nrow(df), m), y.obs := y]
|
||||
} else {
|
||||
df <- df[, y.obs := y]
|
||||
}
|
||||
|
||||
odds.y1 <- qlogis(prediction_accuracy)
|
||||
odds.y0 <- qlogis(prediction_accuracy,lower.tail=F)
|
||||
|
||||
df[y==0,w:=plogis(rlogis(.N,odds.y0))]
|
||||
df[y==1,w:=plogis(rlogis(.N,odds.y1))]
|
||||
|
||||
df[,w_pred := as.integer(w > 0.5)]
|
||||
|
||||
print(mean(df[x==0]$y == df[x==0]$w_pred))
|
||||
print(mean(df[x==1]$y == df[x==1]$w_pred))
|
||||
print(mean(df$w_pred == df$y))
|
||||
return(df)
|
||||
}
|
||||
|
||||
parser <- arg_parser("Simulate data and fit corrected models")
|
||||
parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.72)
|
||||
## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
|
||||
## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
|
||||
parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
|
||||
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
|
||||
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y")
|
||||
|
||||
args <- parse_args(parser)
|
||||
|
||||
B0 <- 0
|
||||
Bxy <- args$Bxy
|
||||
Bzy <- args$Bzy
|
||||
|
||||
|
||||
if(args$m < args$N){
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy)
|
||||
|
||||
# result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
|
||||
|
||||
outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula))
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
}
|
||||
@ -31,13 +31,14 @@ source("simulation_base.R")
|
||||
|
||||
## one way to do it is by adding correlation to x.obs and y that isn't in w.
|
||||
## in other words, the model is missing an important feature of x.obs that's related to y.
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, accuracy_imbalance_difference=0.3){
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, x_bias=-0.75){
|
||||
set.seed(seed)
|
||||
|
||||
# make w and y dependent
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, 0.5)
|
||||
|
||||
ystar <- Bzy * z + Bxy * x
|
||||
ystar <- Bzy * z + Bxy * x + B0
|
||||
y <- rbinom(N,1,plogis(ystar))
|
||||
|
||||
# glm(y ~ x + z, family="binomial")
|
||||
@ -49,40 +50,18 @@ simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, ac
|
||||
} else {
|
||||
df <- df[, y.obs := y]
|
||||
}
|
||||
|
||||
df <- df[,w_pred:=y]
|
||||
|
||||
pz <- mean(z)
|
||||
|
||||
accuracy_imbalance_ratio <- (prediction_accuracy + accuracy_imbalance_difference/2) / (prediction_accuracy - accuracy_imbalance_difference/2)
|
||||
|
||||
# this works because of conditional probability
|
||||
accuracy_z0 <- prediction_accuracy / (pz*(accuracy_imbalance_ratio) + (1-pz))
|
||||
accuracy_z1 <- accuracy_imbalance_ratio * accuracy_z0
|
||||
|
||||
|
||||
yz0 <- df[z==0]$y
|
||||
yz1 <- df[z==1]$y
|
||||
nz1 <- nrow(df[z==1])
|
||||
nz0 <- nrow(df[z==0])
|
||||
|
||||
acc_z0 <- plogis(0.7*scale(yz0) + qlogis(accuracy_z0))
|
||||
acc_z1 <- plogis(1.3*scale(yz1) + qlogis(accuracy_z1))
|
||||
|
||||
w0z0 <- (1-yz0)**2 + (-1)**(1-yz0) * acc_z0
|
||||
w0z1 <- (1-yz1)**2 + (-1)**(1-yz1) * acc_z1
|
||||
|
||||
w0z0.noisy.odds <- rlogis(nz0,qlogis(w0z0))
|
||||
w0z1.noisy.odds <- rlogis(nz1,qlogis(w0z1))
|
||||
df[z==0,w:=plogis(w0z0.noisy.odds)]
|
||||
df[z==1,w:=plogis(w0z1.noisy.odds)]
|
||||
odds.y1 <- qlogis(prediction_accuracy) + x_bias*df[y==1]$x
|
||||
odds.y0 <- qlogis(prediction_accuracy,lower.tail=F) + x_bias*df[y==0]$x
|
||||
|
||||
df[,w_pred:=as.integer(w > 0.5)]
|
||||
df[y==0,w:=plogis(rlogis(.N,odds.y0))]
|
||||
df[y==1,w:=plogis(rlogis(.N,odds.y1))]
|
||||
|
||||
print(mean(df[y==0]$y == df[y==0]$w_pred))
|
||||
print(mean(df[y==1]$y == df[y==1]$w_pred))
|
||||
df[,w_pred := as.integer(w > 0.5)]
|
||||
|
||||
print(mean(df[x==0]$y == df[x==0]$w_pred))
|
||||
print(mean(df[x==1]$y == df[x==1]$w_pred))
|
||||
print(mean(df$w_pred == df$y))
|
||||
|
||||
return(df)
|
||||
}
|
||||
|
||||
@ -92,21 +71,29 @@ parser <- add_argument(parser, "--m", default=500, help="m the number of ground
|
||||
parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.73)
|
||||
parser <- add_argument(parser, "--accuracy_imbalance_difference", help='how much more accurate is the predictive model for one class than the other?', default=0.3)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.8)
|
||||
## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
|
||||
## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
|
||||
parser <- add_argument(parser, "--x_bias", help='how is the classifier biased?', default=0.75)
|
||||
parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
|
||||
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
|
||||
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y*x")
|
||||
|
||||
args <- parse_args(parser)
|
||||
|
||||
B0 <- 0
|
||||
Bxy <- 0.7
|
||||
Bzy <- -0.7
|
||||
Bxy <- args$Bxy
|
||||
Bzy <- args$Bzy
|
||||
|
||||
|
||||
if(args$m < args$N){
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$accuracy_imbalance_difference)
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$x_bias_y0, args$x_bias_y1)
|
||||
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'accuracy_imbalance_difference'=args$accuracy_imbalance_difference)
|
||||
# result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias'=args$x_bias,'x_bias'=args$x_bias,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
|
||||
|
||||
outline <- run_simulation_depvar(df, result, outcome_formula = y ~ x + z, proxy_formula = w_pred ~ y*x + y*z + z*x)
|
||||
outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula))
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
|
||||
|
||||
110
simulations/04_depvar_differential.R
Normal file
110
simulations/04_depvar_differential.R
Normal file
@ -0,0 +1,110 @@
|
||||
### EXAMPLE 1: demonstrates how measurement error can lead to a type sign error in a covariate
|
||||
### What kind of data invalidates fong + tyler?
|
||||
### Even when you have a good predictor, if it's biased against a covariate you can get the wrong sign.
|
||||
### Even when you include the proxy variable in the regression.
|
||||
### But with some ground truth and multiple imputation, you can fix it.
|
||||
|
||||
library(argparser)
|
||||
library(mecor)
|
||||
library(ggplot2)
|
||||
library(data.table)
|
||||
library(filelock)
|
||||
library(arrow)
|
||||
library(Amelia)
|
||||
library(Zelig)
|
||||
library(predictionError)
|
||||
options(amelia.parallel="no",
|
||||
amelia.ncpus=1)
|
||||
setDTthreads(40)
|
||||
|
||||
source("simulation_base.R")
|
||||
|
||||
## SETUP:
|
||||
### we want to estimate x -> y; x is MAR
|
||||
### we have x -> k; k -> w; x -> w is used to predict x via the model w.
|
||||
### A realistic scenario is that we have an NLP model predicting something like "racial harassment" in social media comments
|
||||
### The labels x are binary, but the model provides a continuous predictor
|
||||
|
||||
### simulation:
|
||||
#### how much power do we get from the model in the first place? (sweeping N and m)
|
||||
####
|
||||
|
||||
## one way to do it is by adding correlation to x.obs and y that isn't in w.
|
||||
## in other words, the model is missing an important feature of x.obs that's related to y.
|
||||
simulate_data <- function(N, m, B0, Bxy, Bzy, seed, prediction_accuracy=0.73, x_bias=-0.75){
|
||||
set.seed(seed)
|
||||
|
||||
# make w and y dependent
|
||||
z <- rbinom(N, 1, 0.5)
|
||||
x <- rbinom(N, 1, 0.5)
|
||||
|
||||
ystar <- Bzy * z + Bxy * x + B0
|
||||
y <- rbinom(N,1,plogis(ystar))
|
||||
|
||||
# glm(y ~ x + z, family="binomial")
|
||||
|
||||
df <- data.table(x=x,y=y,ystar=ystar,z=z)
|
||||
|
||||
if(m < N){
|
||||
df <- df[sample(nrow(df), m), y.obs := y]
|
||||
} else {
|
||||
df <- df[, y.obs := y]
|
||||
}
|
||||
|
||||
odds.y1 <- qlogis(prediction_accuracy) + x_bias*df[y==1]$x
|
||||
odds.y0 <- qlogis(prediction_accuracy,lower.tail=F) + x_bias*df[y==0]$x
|
||||
|
||||
df[y==0,w:=plogis(rlogis(.N,odds.y0))]
|
||||
df[y==1,w:=plogis(rlogis(.N,odds.y1))]
|
||||
|
||||
df[,w_pred := as.integer(w > 0.5)]
|
||||
|
||||
print(mean(df[x==0]$y == df[x==0]$w_pred))
|
||||
print(mean(df[x==1]$y == df[x==1]$w_pred))
|
||||
print(mean(df$w_pred == df$y))
|
||||
return(df)
|
||||
}
|
||||
|
||||
parser <- arg_parser("Simulate data and fit corrected models")
|
||||
parser <- add_argument(parser, "--N", default=1000, help="number of observations of w")
|
||||
parser <- add_argument(parser, "--m", default=500, help="m the number of ground truth observations")
|
||||
parser <- add_argument(parser, "--seed", default=17, help='seed for the rng')
|
||||
parser <- add_argument(parser, "--outfile", help='output file', default='example_2.feather')
|
||||
parser <- add_argument(parser, "--y_explained_variance", help='what proportion of the variance of y can be explained?', default=0.005)
|
||||
parser <- add_argument(parser, "--prediction_accuracy", help='how accurate is the predictive model?', default=0.8)
|
||||
## parser <- add_argument(parser, "--x_bias_y1", help='how is the classifier biased when y = 1?', default=-0.75)
|
||||
## parser <- add_argument(parser, "--x_bias_y0", help='how is the classifier biased when y = 0 ?', default=0.75)
|
||||
parser <- add_argument(parser, "--x_bias", help='how is the classifier biased?', default=0.75)
|
||||
parser <- add_argument(parser, "--Bxy", help='coefficient of x on y', default=0.3)
|
||||
parser <- add_argument(parser, "--Bzy", help='coeffficient of z on y', default=-0.3)
|
||||
parser <- add_argument(parser, "--outcome_formula", help='formula for the outcome variable', default="y~x+z")
|
||||
parser <- add_argument(parser, "--proxy_formula", help='formula for the proxy variable', default="w_pred~y+x")
|
||||
|
||||
args <- parse_args(parser)
|
||||
|
||||
B0 <- 0
|
||||
Bxy <- args$Bxy
|
||||
Bzy <- args$Bzy
|
||||
|
||||
|
||||
if(args$m < args$N){
|
||||
df <- simulate_data(args$N, args$m, B0, Bxy, Bzy, args$seed, args$prediction_accuracy, args$x_bias)
|
||||
|
||||
# result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias_y0'=args$x_bias_y0,'x_bias_y1'=args$x_bias_y1,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
|
||||
result <- list('N'=args$N,'m'=args$m,'B0'=B0,'Bxy'=Bxy,'Bzy'=Bzy, 'seed'=args$seed, 'y_explained_variance'=args$y_explained_variance, 'prediction_accuracy'=args$prediction_accuracy, 'x_bias'=args$x_bias,'outcome_formula' = args$outcome_formula, 'proxy_formula' = args$proxy_formula)
|
||||
|
||||
outline <- run_simulation_depvar(df, result, outcome_formula = as.formula(args$outcome_formula), proxy_formula = as.formula(args$proxy_formula))
|
||||
|
||||
outfile_lock <- lock(paste0(args$outfile, '_lock'),exclusive=TRUE)
|
||||
|
||||
if(file.exists(args$outfile)){
|
||||
logdata <- read_feather(args$outfile)
|
||||
logdata <- rbind(logdata,as.data.table(outline),fill=TRUE)
|
||||
} else {
|
||||
logdata <- as.data.table(outline)
|
||||
}
|
||||
|
||||
print(outline)
|
||||
write_feather(logdata, args$outfile)
|
||||
unlock(outfile_lock)
|
||||
}
|
||||
@ -1,9 +1,9 @@
|
||||
|
||||
SHELL=bash
|
||||
|
||||
Ns=[1000,3600,14400]
|
||||
ms=[75,150,300]
|
||||
seeds=[$(shell seq -s, 1 250)]
|
||||
Ns=[1000, 2000, 4000, 8000]
|
||||
ms=[100, 200, 400, 800]
|
||||
seeds=[$(shell seq -s, 1 100)]
|
||||
explained_variances=[0.1]
|
||||
|
||||
all:remembr.RDS
|
||||
@ -31,7 +31,7 @@ example_1.feather: example_1_jobs
|
||||
# sbatch --wait --verbose --array=3001-6001 run_simulation.sbatch 0 example_1_jobs
|
||||
|
||||
example_2_jobs: 02_indep_differential.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "accuracy_imbalance_difference":[0.3], "Bzy":[0.3]}' --outfile example_2_jobs
|
||||
grid_sweep.py --command "Rscript 02_indep_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_2.feather"],"y_explained_variance":${explained_variances}, "Bzy":[-0.3],"Bxy":[0.3],"Bzx":[0.3], "outcome_formula":["y~x+z"], "proxy_formula":["w_pred~y*z*x"], "truth_formula":["x~z"]}' --outfile example_2_jobs
|
||||
|
||||
example_2.feather: example_2_jobs
|
||||
rm -f example_2.feather
|
||||
@ -45,19 +45,26 @@ example_2.feather: example_2_jobs
|
||||
# rm -f example_2_B.feather
|
||||
# sbatch --wait --verbose --array=1-3000 run_simulation.sbatch 0 example_2_B_jobs
|
||||
|
||||
example_3_jobs: 03_depvar_differential.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 03_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
|
||||
example_3_jobs: 03_depvar.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 03_depvar.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_3.feather"], "y_explained_variance":${explained_variances}}' --outfile example_3_jobs
|
||||
|
||||
example_3.feather: example_3_jobs
|
||||
rm -f example_3.feather
|
||||
sbatch --wait --verbose --array=1-$(shell cat example_3_jobs | wc -l) run_simulation.sbatch 0 example_3_jobs
|
||||
|
||||
example_4_jobs: 04_depvar_differential.R simulation_base.R
|
||||
grid_sweep.py --command "Rscript 04_depvar_differential.R" --arg_dict '{"N":${Ns},"m":${ms}, "seed":${seeds}, "outfile":["example_4.feather"], "y_explained_variance":${explained_variances}}' --outfile example_4_jobs
|
||||
|
||||
remembr.RDS:example_1.feather example_2.feather example_3.feather plot_example.R plot_dv_example.R
|
||||
example_4.feather: example_4_jobs
|
||||
rm -f example_4.feather
|
||||
sbatch --wait --verbose --array=1-$(shell cat example_4_jobs | wc -l) run_simulation.sbatch 0 example_4_jobs
|
||||
|
||||
remembr.RDS:example_1.feather example_2.feather example_3.feather example_4.feather plot_example.R plot_dv_example.R
|
||||
rm -f remembr.RDS
|
||||
${srun} Rscript plot_example.R --infile example_1.feather --name "plot.df.example.1"
|
||||
${srun} Rscript plot_example.R --infile example_2.feather --name "plot.df.example.2"
|
||||
${srun} Rscript plot_dv_example.R --infile example_3.feather --name "plot.df.example.3"
|
||||
${srun} Rscript plot_dv_example.R --infile example_4.feather --name "plot.df.example.4"
|
||||
|
||||
clean:
|
||||
rm *.feather
|
||||
|
||||
@ -57,7 +57,7 @@ measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='lo
|
||||
df.unobs.y1 <- copy(df.unobs)
|
||||
df.unobs.y1[[response.var]] <- 1
|
||||
df.unobs.y0 <- copy(df.unobs)
|
||||
df.unobs.y0[[response.var]] <- 1
|
||||
df.unobs.y0[[response.var]] <- 0
|
||||
|
||||
## integrate out y
|
||||
outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
|
||||
@ -124,6 +124,8 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
|
||||
if(outcome_family$family == "gaussian"){
|
||||
sigma.y <- params[param.idx]
|
||||
param.idx <- param.idx + 1
|
||||
|
||||
# outcome_formula likelihood using linear regression
|
||||
ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
|
||||
}
|
||||
|
||||
@ -135,6 +137,8 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
|
||||
|
||||
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
||||
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
|
||||
|
||||
# proxy_formula likelihood using logistic regression
|
||||
ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
|
||||
ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
@ -149,10 +153,13 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
|
||||
|
||||
if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
|
||||
ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
|
||||
|
||||
# truth_formula likelihood using logistic regression
|
||||
ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
|
||||
ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
|
||||
# add the three likelihoods
|
||||
ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
|
||||
|
||||
## likelihood for the predicted data
|
||||
@ -169,6 +176,8 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
|
||||
outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
|
||||
outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
|
||||
if(outcome_family$family=="gaussian"){
|
||||
|
||||
# likelihood of outcome
|
||||
ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
|
||||
ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
|
||||
}
|
||||
@ -181,6 +190,7 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
|
||||
ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
|
||||
ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
|
||||
|
||||
# likelihood of proxy
|
||||
ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
|
||||
ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
|
||||
|
||||
@ -190,8 +200,9 @@ measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_fo
|
||||
|
||||
if(truth_family$link=='logit'){
|
||||
truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
|
||||
ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
|
||||
ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
|
||||
# likelihood of truth
|
||||
ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
|
||||
ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -21,8 +21,7 @@ summarize.estimator <- function(df, suffix='naive', coefname='x'){
|
||||
paste0('B',coefname,'y.ci.lower.',suffix),
|
||||
paste0('B',coefname,'y.ci.upper.',suffix),
|
||||
'y_explained_variance',
|
||||
'Bzy',
|
||||
'accuracy_imbalance_difference'
|
||||
'Bzy'
|
||||
),
|
||||
with=FALSE]
|
||||
|
||||
@ -47,7 +46,7 @@ summarize.estimator <- function(df, suffix='naive', coefname='x'){
|
||||
variable=coefname,
|
||||
method=suffix
|
||||
),
|
||||
by=c("N","m",'Bzy','accuracy_imbalance_difference','y_explained_variance')
|
||||
by=c("N","m",'Bzy','y_explained_variance')
|
||||
]
|
||||
|
||||
return(part.plot)
|
||||
|
||||
@ -99,6 +99,7 @@ build_plot_dataset <- function(df){
|
||||
|
||||
|
||||
plot.df <- read_feather(args$infile)
|
||||
print(unique(plot.df$N))
|
||||
|
||||
# df <- df[apply(df,1,function(x) !any(is.na(x)))]
|
||||
|
||||
|
||||
@ -41,21 +41,26 @@ my.pseudo.mle <- function(df){
|
||||
## Zhang got this model from Hausman 1998
|
||||
### I think this is actually eqivalent to the pseudo.mle method
|
||||
zhang.mle.iv <- function(df){
|
||||
nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1, ppv=0.9, npv=0.9){
|
||||
df.obs <- df[!is.na(x.obs)]
|
||||
df.unobs <- df[is.na(x.obs)]
|
||||
|
||||
tn <- df.obs[(w_pred == 0) & (x.obs == w_pred),.N]
|
||||
pn <- df.obs[(w_pred==0), .N]
|
||||
npv <- tn / pn
|
||||
|
||||
tp <- df.obs[(w_pred==1) & (x.obs == w_pred),.N]
|
||||
pp <- df.obs[(w_pred==1),.N]
|
||||
ppv <- tp / pp
|
||||
|
||||
nll <- function(B0=0, Bxy=0, Bzy=0, sigma_y=0.1){
|
||||
|
||||
## fpr = 1 - TNR
|
||||
### Problem: accounting for uncertainty in ppv / npv
|
||||
|
||||
ll.w1x1.obs <- with(df.obs[(w_pred==1)], dbinom(x.obs,size=1,prob=ppv,log=T))
|
||||
ll.w0x0.obs <- with(df.obs[(w_pred==0)], dbinom(1-x.obs,size=1,prob=npv,log=T))
|
||||
|
||||
## fnr = 1 - TPR
|
||||
ll.y.obs <- with(df.obs, dnorm(y, B0 + Bxy * x + Bzy * z, sd=sigma_y,log=T))
|
||||
ll <- sum(ll.y.obs)
|
||||
ll <- ll + sum(ll.w1x1.obs) + sum(ll.w0x0.obs)
|
||||
|
||||
|
||||
# unobserved case; integrate out x
|
||||
ll.x.1 <- with(df.unobs, dnorm(y, B0 + Bxy + Bzy * z, sd = sigma_y, log=T))
|
||||
ll.x.0 <- with(df.unobs, dnorm(y, B0 + Bzy * z, sd = sigma_y,log=T))
|
||||
@ -66,55 +71,90 @@ zhang.mle.iv <- function(df){
|
||||
## case x == 0
|
||||
lls.x.0 <- colLogSumExps(rbind(log(1-npv) + ll.x.1, log(npv) + ll.x.0))
|
||||
|
||||
lls <- colLogSumExps(rbind(lls.x.1, lls.x.0))
|
||||
lls <- colLogSumExps(rbind(df.unobs$w_pred * lls.x.1, (1-df.unobs$w_pred) * lls.x.0))
|
||||
ll <- ll + sum(lls)
|
||||
return(-ll)
|
||||
}
|
||||
mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf,ppv=0.00001, npv=0.00001),
|
||||
upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf, ppv=0.99999,npv=0.99999),method='L-BFGS-B')
|
||||
mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6), lower=list(sigma_y=0.0001, B0=-Inf, Bxy=-Inf, Bzy=-Inf),
|
||||
upper=list(sigma_y=Inf, B0=Inf, Bxy=Inf, Bzy=Inf),method='L-BFGS-B')
|
||||
return(mlefit)
|
||||
}
|
||||
|
||||
## this is equivalent to the pseudo-liklihood model from Carolla
|
||||
zhang.mle.dv <- function(df){
|
||||
## this is equivalent to the pseudo-liklihood model from Caroll
|
||||
## zhang.mle.dv <- function(df){
|
||||
|
||||
nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){
|
||||
df.obs <- df[!is.na(y.obs)]
|
||||
## nll <- function(B0=0, Bxy=0, Bzy=0, ppv=0.9, npv=0.9){
|
||||
## df.obs <- df[!is.na(y.obs)]
|
||||
|
||||
## fpr = 1 - TNR
|
||||
ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE))
|
||||
ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE))
|
||||
## ## fpr = 1 - TNR
|
||||
## ll.w0y0 <- with(df.obs[y.obs==0],dbinom(1-w_pred,1,npv,log=TRUE))
|
||||
## ll.w1y1 <- with(df.obs[y.obs==1],dbinom(w_pred,1,ppv,log=TRUE))
|
||||
|
||||
# observed case
|
||||
ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
|
||||
ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
|
||||
ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
|
||||
## # observed case
|
||||
## ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
|
||||
## ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
|
||||
## ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
|
||||
|
||||
ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1)
|
||||
## ll <- sum(ll.y.obs) + sum(ll.w0y0) + sum(ll.w1y1)
|
||||
|
||||
# unobserved case; integrate out y
|
||||
## case y = 1
|
||||
ll.y.1 <- vector(mode='numeric', length=nrow(df))
|
||||
pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
|
||||
## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1)
|
||||
lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1))
|
||||
## # unobserved case; integrate out y
|
||||
## ## case y = 1
|
||||
## ll.y.1 <- vector(mode='numeric', length=nrow(df))
|
||||
## pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
|
||||
## ## P(w=1| y=1)P(y=1) + P(w=0|y=1)P(y=1) = P(w=1,y=1) + P(w=0,y=1)
|
||||
## lls.y.1 <- colLogSumExps(rbind(log(ppv) + pi.y.1, log(1-ppv) + pi.y.1))
|
||||
|
||||
## case y = 0
|
||||
ll.y.0 <- vector(mode='numeric', length=nrow(df))
|
||||
pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
|
||||
## ## case y = 0
|
||||
## ll.y.0 <- vector(mode='numeric', length=nrow(df))
|
||||
## pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
|
||||
|
||||
## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0)
|
||||
lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0))
|
||||
## ## P(w=1 | y=0)P(y=0) + P(w=0|y=0)P(y=0) = P(w=1,y=0) + P(w=0,y=0)
|
||||
## lls.y.0 <- colLogSumExps(rbind(log(npv) + pi.y.0, log(1-npv) + pi.y.0))
|
||||
|
||||
lls <- colLogSumExps(rbind(lls.y.1, lls.y.0))
|
||||
ll <- ll + sum(lls)
|
||||
return(-ll)
|
||||
## lls <- colLogSumExps(rbind(lls.y.1, lls.y.0))
|
||||
## ll <- ll + sum(lls)
|
||||
## return(-ll)
|
||||
## }
|
||||
## mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001),
|
||||
## upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999))
|
||||
## return(mlefit)
|
||||
## }
|
||||
|
||||
zhang.mle.dv <- function(df){
|
||||
df.obs <- df[!is.na(y.obs)]
|
||||
df.unobs <- df[is.na(y.obs)]
|
||||
|
||||
fp <- df.obs[(w_pred==1) & (y.obs != w_pred),.N]
|
||||
p <- df.obs[(w_pred==1),.N]
|
||||
fpr <- fp / p
|
||||
fn <- df.obs[(w_pred==0) & (y.obs != w_pred), .N]
|
||||
n <- df.obs[(w_pred==0),.N]
|
||||
fnr <- fn / n
|
||||
|
||||
nll <- function(B0=0, Bxy=0, Bzy=0){
|
||||
|
||||
|
||||
## observed case
|
||||
ll.y.obs <- vector(mode='numeric', length=nrow(df.obs))
|
||||
ll.y.obs[df.obs$y.obs==1] <- with(df.obs[y.obs==1], plogis(B0 + Bxy * x + Bzy * z,log=T))
|
||||
ll.y.obs[df.obs$y.obs==0] <- with(df.obs[y.obs==0], plogis(B0 + Bxy * x + Bzy * z,log=T,lower.tail=FALSE))
|
||||
|
||||
ll <- sum(ll.y.obs)
|
||||
|
||||
pi.y.1 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T))
|
||||
pi.y.0 <- with(df,plogis(B0 + Bxy * x + Bzy*z, log=T,lower.tail=FALSE))
|
||||
|
||||
lls <- with(df.unobs, colLogSumExps(rbind(w_pred * colLogSumExps(rbind(log(fpr), log(1 - fnr - fpr)+pi.y.1)),
|
||||
(1-w_pred) * colLogSumExps(rbind(log(1-fpr), log(1 - fnr - fpr)+pi.y.0)))))
|
||||
|
||||
ll <- ll + sum(lls)
|
||||
return(-ll)
|
||||
}
|
||||
mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=list(B0=-Inf, Bxy=-Inf, Bzy=-Inf, ppv=0.001,npv=0.001),
|
||||
upper=list(B0=Inf, Bxy=Inf, Bzy=Inf,ppv=0.999,npv=0.999))
|
||||
mlefit <- mle2(minuslogl = nll, control=list(maxit=1e6),method='L-BFGS-B',lower=c(B0=-Inf, Bxy=-Inf, Bzy=-Inf),
|
||||
upper=c(B0=Inf, Bxy=Inf, Bzy=Inf))
|
||||
return(mlefit)
|
||||
}
|
||||
|
||||
|
||||
## This uses the likelihood approach from Carroll page 353.
|
||||
## assumes that we have a good measurement error model
|
||||
my.mle <- function(df){
|
||||
@ -211,7 +251,7 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
|
||||
naivecont.ci.Bxy <- confint(model.naive.cont)['x',]
|
||||
naivecont.ci.Bzy <- confint(model.naive.cont)['z',]
|
||||
|
||||
## my implementatoin of liklihood based correction
|
||||
## my implementation of liklihood based correction
|
||||
|
||||
temp.df <- copy(df)
|
||||
temp.df[,y:=y.obs]
|
||||
@ -241,7 +281,8 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
|
||||
Bzy.est.zhang = coef['Bzy'],
|
||||
Bzy.ci.upper.zhang = ci['Bzy','97.5 %'],
|
||||
Bzy.ci.lower.zhang = ci['Bzy','2.5 %']))
|
||||
|
||||
|
||||
|
||||
|
||||
# amelia says use normal distribution for binary variables.
|
||||
tryCatch({
|
||||
@ -278,7 +319,7 @@ run_simulation_depvar <- function(df, result, outcome_formula=y~x+z, proxy_formu
|
||||
|
||||
|
||||
## outcome_formula, proxy_formula, and truth_formula are passed to measerr_mle
|
||||
run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_pred~x, truth_formula=x~z){
|
||||
run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=NULL, truth_formula=NULL){
|
||||
|
||||
accuracy <- df[,mean(w_pred==x)]
|
||||
result <- append(result, list(accuracy=accuracy))
|
||||
@ -320,7 +361,7 @@ run_simulation <- function(df, result, outcome_formula=y~x+z, proxy_formula=w_p
|
||||
|
||||
|
||||
tryCatch({
|
||||
amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w_pred'))
|
||||
amelia.out.k <- amelia(df, m=200, p2s=0, idvars=c('x','w'))
|
||||
mod.amelia.k <- zelig(y~x.obs+z, model='ls', data=amelia.out.k$imputations, cite=FALSE)
|
||||
(coefse <- combine_coef_se(mod.amelia.k, messages=FALSE))
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user