471 lines
23 KiB
R
471 lines
23 KiB
R
library(formula.tools)
|
|
library(matrixStats)
|
|
library(bbmle)
|
|
## df: dataframe to model
|
|
## outcome_formula: formula for y | x, z
|
|
## outcome_family: family for y | x, z
|
|
## proxy_formula: formula for w | x, z, y
|
|
## proxy_family: family for w | x, z, y
|
|
## truth_formula: formula for x | z
|
|
## truth_family: family for x | z
|
|
|
|
### ideal formulas for example 1
|
|
# test.fit.1 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x, binomial(link='logit'), x ~ z)
|
|
|
|
### ideal formulas for example 2
|
|
# test.fit.2 <- measerr_mle(df, y ~ x + z, gaussian(), w_pred ~ x + z + y + y:x, binomial(link='logit'), x ~ z)
|
|
|
|
|
|
## outcome_formula <- y ~ x + z; proxy_formula <- w_pred ~ y + x + z + x:z + x:y + z:y
|
|
measerr_mle_dv <- function(df, outcome_formula, outcome_family=binomial(link='logit'), proxy_formula, proxy_family=binomial(link='logit'),method='optim'){
|
|
|
|
nll <- function(params){
|
|
df.obs <- model.frame(outcome_formula, df)
|
|
proxy.variable <- all.vars(proxy_formula)[1]
|
|
proxy.model.matrix <- model.matrix(proxy_formula, df)
|
|
response.var <- all.vars(outcome_formula)[1]
|
|
y.obs <- with(df.obs,eval(parse(text=response.var)))
|
|
outcome.model.matrix <- model.matrix(outcome_formula, df.obs)
|
|
|
|
param.idx <- 1
|
|
n.outcome.model.covars <- dim(outcome.model.matrix)[2]
|
|
outcome.params <- params[param.idx:n.outcome.model.covars]
|
|
param.idx <- param.idx + n.outcome.model.covars
|
|
|
|
if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
|
|
ll.y.obs <- vector(mode='numeric', length=length(y.obs))
|
|
ll.y.obs[y.obs==1] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==1,]),log=TRUE)
|
|
ll.y.obs[y.obs==0] <- plogis(outcome.params %*% t(outcome.model.matrix[y.obs==0,]),log=TRUE,lower.tail=FALSE)
|
|
}
|
|
|
|
df.obs <- model.frame(proxy_formula,df)
|
|
n.proxy.model.covars <- dim(proxy.model.matrix)[2]
|
|
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
|
|
|
|
param.idx <- param.idx + n.proxy.model.covars
|
|
proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
|
|
|
|
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
|
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
|
|
ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
|
|
ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
|
}
|
|
|
|
ll.obs <- sum(ll.y.obs + ll.w.obs)
|
|
|
|
df.unobs <- df[is.na(df[[response.var]])]
|
|
df.unobs.y1 <- copy(df.unobs)
|
|
df.unobs.y1[[response.var]] <- 1
|
|
df.unobs.y0 <- copy(df.unobs)
|
|
df.unobs.y0[[response.var]] <- 0
|
|
|
|
## integrate out y
|
|
outcome.model.matrix.y1 <- model.matrix(outcome_formula, df.unobs.y1)
|
|
|
|
if((outcome_family$family == "binomial") & (outcome_family$link == 'logit')){
|
|
ll.y.unobs.1 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
|
|
ll.y.unobs.0 <- vector(mode='numeric', length=dim(outcome.model.matrix.y1)[1])
|
|
ll.y.unobs.1 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE)
|
|
ll.y.unobs.0 <- plogis(outcome.params %*% t(outcome.model.matrix.y1),log=TRUE,lower.tail=FALSE)
|
|
}
|
|
|
|
proxy.model.matrix.y1 <- model.matrix(proxy_formula, df.unobs.y1)
|
|
proxy.model.matrix.y0 <- model.matrix(proxy_formula, df.unobs.y0)
|
|
proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
|
|
|
|
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
|
ll.w.unobs.1 <- vector(mode='numeric',length=dim(proxy.model.matrix.y1)[1])
|
|
ll.w.unobs.0 <- vector(mode='numeric',length=dim(proxy.model.matrix.y0)[1])
|
|
ll.w.unobs.1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==1,]),log=TRUE)
|
|
ll.w.unobs.1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y1[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
|
|
|
|
ll.w.unobs.0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==1,]),log=TRUE)
|
|
ll.w.unobs.0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.y0[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
|
|
}
|
|
|
|
ll.unobs.1 <- ll.y.unobs.1 + ll.w.unobs.1
|
|
ll.unobs.0 <- ll.y.unobs.0 + ll.w.unobs.0
|
|
ll.unobs <- sum(colLogSumExps(rbind(ll.unobs.1,ll.unobs.0)))
|
|
ll <- ll.unobs + ll.obs
|
|
return(-ll)
|
|
}
|
|
|
|
params <- colnames(model.matrix(outcome_formula,df))
|
|
lower <- rep(-Inf, length(params))
|
|
proxy.params <- colnames(model.matrix(proxy_formula, df))
|
|
params <- c(params, paste0('proxy_',proxy.params))
|
|
lower <- c(lower, rep(-Inf, length(proxy.params)))
|
|
start <- rep(0.1,length(params))
|
|
names(start) <- params
|
|
|
|
if(method=='optim'){
|
|
fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
|
|
} else {
|
|
quoted.names <- gsub("[\\(\\)]",'',names(start))
|
|
print(quoted.names)
|
|
text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
|
|
|
|
measerr_mle_nll <- eval(parse(text=text))
|
|
names(start) <- quoted.names
|
|
names(lower) <- quoted.names
|
|
fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
|
|
}
|
|
return(fit)
|
|
}
|
|
|
|
## Experimental, and not necessary if errors are independent.
|
|
measerr_irr_mle <- function(df, outcome_formula, outcome_family=gaussian(), rater_formula, proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
|
|
|
|
### in this scenario, the ground truth also has measurement error, but we have repeated measures for it.
|
|
|
|
## probability of y given observed data.
|
|
df.obs <- df[!is.na(x.obs.1)]
|
|
proxy.variable <- all.vars(proxy_formula)[1]
|
|
df.x.obs.1 <- copy(df.obs)[,x:=1]
|
|
df.x.obs.0 <- copy(df.obs)[,x:=0]
|
|
y.obs <- df.obs[,y]
|
|
|
|
nll <- function(params){
|
|
outcome.model.matrix.x.obs.0 <- model.matrix(outcome_formula, df.x.obs.0)
|
|
outcome.model.matrix.x.obs.1 <- model.matrix(outcome_formula, df.x.obs.1)
|
|
|
|
param.idx <- 1
|
|
n.outcome.model.covars <- dim(outcome.model.matrix.x.obs.0)[2]
|
|
outcome.params <- params[param.idx:n.outcome.model.covars]
|
|
param.idx <- param.idx + n.outcome.model.covars
|
|
|
|
sigma.y <- params[param.idx]
|
|
param.idx <- param.idx + 1
|
|
|
|
ll.y.x.obs.0 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.0),sd=sigma.y, log=TRUE)
|
|
ll.y.x.obs.1 <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix.x.obs.1),sd=sigma.y, log=TRUE)
|
|
|
|
## assume that the two coders are statistically independent conditional on x
|
|
ll.x.obs.0.x0 <- vector(mode='numeric', length=nrow(df.obs))
|
|
ll.x.obs.1.x0 <- vector(mode='numeric', length=nrow(df.obs))
|
|
ll.x.obs.0.x1 <- vector(mode='numeric', length=nrow(df.obs))
|
|
ll.x.obs.1.x1 <- vector(mode='numeric', length=nrow(df.obs))
|
|
|
|
rater.model.matrix.x.obs.0 <- model.matrix(rater_formula, df.x.obs.0)
|
|
rater.model.matrix.x.obs.1 <- model.matrix(rater_formula, df.x.obs.1)
|
|
|
|
n.rater.model.covars <- dim(rater.model.matrix.x.obs.0)[2]
|
|
rater.0.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
|
|
param.idx <- param.idx + n.rater.model.covars
|
|
|
|
rater.1.params <- params[param.idx:(n.rater.model.covars + param.idx - 1)]
|
|
param.idx <- param.idx + n.rater.model.covars
|
|
|
|
# probability of rater 0 if x is 0 or 1
|
|
ll.x.obs.0.x0[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==1,]), log=TRUE)
|
|
ll.x.obs.0.x0[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
|
|
ll.x.obs.0.x1[df.obs$x.obs.0==1] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==1,]), log=TRUE)
|
|
ll.x.obs.0.x1[df.obs$x.obs.0==0] <- plogis(rater.0.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.0==0,]), log=TRUE, lower.tail=FALSE)
|
|
|
|
# probability of rater 1 if x is 0 or 1
|
|
ll.x.obs.1.x0[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==1,]), log=TRUE)
|
|
ll.x.obs.1.x0[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.0[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
|
|
ll.x.obs.1.x1[df.obs$x.obs.1==1] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==1,]), log=TRUE)
|
|
ll.x.obs.1.x1[df.obs$x.obs.1==0] <- plogis(rater.1.params %*% t(rater.model.matrix.x.obs.1[df.obs$x.obs.1==0,]), log=TRUE, lower.tail=FALSE)
|
|
|
|
proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.x.obs.0)
|
|
proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.x.obs.1)
|
|
|
|
n.proxy.model.covars <- dim(proxy.model.matrix.x0)[2]
|
|
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
|
|
param.idx <- param.idx + n.proxy.model.covars
|
|
|
|
proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
|
|
|
|
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
|
ll.w.obs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
|
|
ll.w.obs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
|
|
|
|
# proxy_formula likelihood using logistic regression
|
|
ll.w.obs.x0[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==1,]),log=TRUE)
|
|
ll.w.obs.x0[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
|
|
|
ll.w.obs.x1[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==1,]),log=TRUE)
|
|
ll.w.obs.x1[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
|
}
|
|
|
|
## assume that the probability of x is a logistic regression depending on z
|
|
truth.model.matrix.obs <- model.matrix(truth_formula, df.obs)
|
|
n.truth.params <- dim(truth.model.matrix.obs)[2]
|
|
truth.params <- params[param.idx:(n.truth.params + param.idx - 1)]
|
|
|
|
ll.obs.x0 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE)
|
|
ll.obs.x1 <- plogis(truth.params %*% t(truth.model.matrix.obs), log=TRUE, lower.tail=FALSE)
|
|
|
|
ll.obs <- colLogSumExps(rbind(ll.y.x.obs.0 + ll.x.obs.0.x0 + ll.x.obs.1.x0 + ll.obs.x0 + ll.w.obs.x0,
|
|
ll.y.x.obs.1 + ll.x.obs.0.x1 + ll.x.obs.1.x1 + ll.obs.x1 + ll.w.obs.x1))
|
|
|
|
### NOW FOR THE FUN PART. Likelihood of the unobserved data.
|
|
### we have to integrate out x.obs.0, x.obs.1, and x.
|
|
|
|
|
|
## THE OUTCOME
|
|
df.unobs <- df[is.na(x.obs)]
|
|
df.x.unobs.0 <- copy(df.unobs)[,x:=0]
|
|
df.x.unobs.1 <- copy(df.unobs)[,x:=1]
|
|
y.unobs <- df.unobs$y
|
|
|
|
outcome.model.matrix.x.unobs.0 <- model.matrix(outcome_formula, df.x.unobs.0)
|
|
outcome.model.matrix.x.unobs.1 <- model.matrix(outcome_formula, df.x.unobs.1)
|
|
|
|
ll.y.unobs.x0 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.0), sd=sigma.y, log=TRUE)
|
|
ll.y.unobs.x1 <- dnorm(y.unobs, outcome.params %*% t(outcome.model.matrix.x.unobs.1), sd=sigma.y, log=TRUE)
|
|
|
|
|
|
## THE UNLABELED DATA
|
|
|
|
|
|
## assume that the two coders are statistically independent conditional on x
|
|
ll.x.unobs.0.x0 <- vector(mode='numeric', length=nrow(df.unobs))
|
|
ll.x.unobs.1.x0 <- vector(mode='numeric', length=nrow(df.unobs))
|
|
ll.x.unobs.0.x1 <- vector(mode='numeric', length=nrow(df.unobs))
|
|
ll.x.unobs.1.x1 <- vector(mode='numeric', length=nrow(df.unobs))
|
|
|
|
df.x.unobs.0[,x.obs := 1]
|
|
df.x.unobs.1[,x.obs := 1]
|
|
|
|
rater.model.matrix.x.unobs.0 <- model.matrix(rater_formula, df.x.unobs.0)
|
|
rater.model.matrix.x.unobs.1 <- model.matrix(rater_formula, df.x.unobs.1)
|
|
|
|
|
|
## # probability of rater 0 if x is 0 or 1
|
|
## ll.x.unobs.0.x0 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
|
|
## plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
|
|
|
|
## ll.x.unobs.0.x1 <- colLogSumExps(rbind(plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
|
|
## plogis(rater.0.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
|
|
|
|
## # probability of rater 1 if x is 0 or 1
|
|
## ll.x.unobs.1.x0 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE),
|
|
## plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.0), log=TRUE, lower.tail=TRUE)))
|
|
|
|
## ll.x.unobs.1.x1 <- colLogSumExps(rbind(plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE),
|
|
## plogis(rater.1.params %*% t(rater.model.matrix.x.unobs.1), log=TRUE, lower.tail=TRUE)))
|
|
|
|
|
|
proxy.unobs <- with(df.unobs, eval(parse(text=proxy.variable)))
|
|
proxy.model.matrix.x0.unobs <- model.matrix(proxy_formula, df.x.unobs.0)
|
|
proxy.model.matrix.x1.unobs <- model.matrix(proxy_formula, df.x.unobs.1)
|
|
|
|
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
|
ll.w.unobs.x0 <- vector(mode='numeric',length=dim(proxy.model.matrix.x0)[1])
|
|
ll.w.unobs.x1 <- vector(mode='numeric',length=dim(proxy.model.matrix.x1)[1])
|
|
|
|
|
|
# proxy_formula likelihood using logistic regression
|
|
ll.w.unobs.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==1,]),log=TRUE)
|
|
ll.w.unobs.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
|
|
|
|
ll.w.unobs.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==1,]),log=TRUE)
|
|
ll.w.unobs.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1.unobs[proxy.unobs==0,]),log=TRUE, lower.tail=FALSE)
|
|
}
|
|
|
|
truth.model.matrix.unobs <- model.matrix(truth_formula, df.unobs)
|
|
|
|
ll.unobs.x0 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE)
|
|
ll.unobs.x1 <- plogis(truth.params %*% t(truth.model.matrix.unobs), log=TRUE, lower.tail=FALSE)
|
|
|
|
ll.unobs <- colLogSumExps(rbind(ll.unobs.x0 + ll.w.unobs.x0 + ll.y.unobs.x0,
|
|
ll.unobs.x1 + ll.w.unobs.x1 + ll.y.unobs.x1))
|
|
|
|
return(-1 *( sum(ll.obs) + sum(ll.unobs)))
|
|
}
|
|
|
|
outcome.params <- colnames(model.matrix(outcome_formula,df))
|
|
lower <- rep(-Inf, length(outcome.params))
|
|
|
|
if(outcome_family$family=='gaussian'){
|
|
params <- c(outcome.params, 'sigma_y')
|
|
lower <- c(lower, 0.00001)
|
|
} else {
|
|
params <- outcome.params
|
|
}
|
|
|
|
rater.0.params <- colnames(model.matrix(rater_formula,df))
|
|
params <- c(params, paste0('rater_0',rater.0.params))
|
|
lower <- c(lower, rep(-Inf, length(rater.0.params)))
|
|
|
|
rater.1.params <- colnames(model.matrix(rater_formula,df))
|
|
params <- c(params, paste0('rater_1',rater.1.params))
|
|
lower <- c(lower, rep(-Inf, length(rater.1.params)))
|
|
|
|
proxy.params <- colnames(model.matrix(proxy_formula, df))
|
|
params <- c(params, paste0('proxy_',proxy.params))
|
|
lower <- c(lower, rep(-Inf, length(proxy.params)))
|
|
|
|
truth.params <- colnames(model.matrix(truth_formula, df))
|
|
params <- c(params, paste0('truth_', truth.params))
|
|
lower <- c(lower, rep(-Inf, length(truth.params)))
|
|
start <- rep(0.1,length(params))
|
|
names(start) <- params
|
|
|
|
|
|
if(method=='optim'){
|
|
fit <- optim(start, fn = nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
|
|
} else {
|
|
|
|
quoted.names <- gsub("[\\(\\)]",'',names(start))
|
|
print(quoted.names)
|
|
text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(nll(params))}")
|
|
|
|
measerr_mle_nll <- eval(parse(text=text))
|
|
names(start) <- quoted.names
|
|
names(lower) <- quoted.names
|
|
fit <- mle2(minuslogl=measerr_mle_nll, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
|
|
}
|
|
|
|
return(fit)
|
|
}
|
|
|
|
|
|
measerr_mle <- function(df, outcome_formula, outcome_family=gaussian(), proxy_formula, proxy_family=binomial(link='logit'), truth_formula, truth_family=binomial(link='logit'),method='optim'){
|
|
|
|
measerr_mle_nll <- function(params){
|
|
df.obs <- model.frame(outcome_formula, df)
|
|
proxy.variable <- all.vars(proxy_formula)[1]
|
|
proxy.model.matrix <- model.matrix(proxy_formula, df)
|
|
response.var <- all.vars(outcome_formula)[1]
|
|
y.obs <- with(df.obs,eval(parse(text=response.var)))
|
|
|
|
outcome.model.matrix <- model.matrix(outcome_formula, df)
|
|
|
|
param.idx <- 1
|
|
n.outcome.model.covars <- dim(outcome.model.matrix)[2]
|
|
outcome.params <- params[param.idx:n.outcome.model.covars]
|
|
param.idx <- param.idx + n.outcome.model.covars
|
|
|
|
## likelihood for the fully observed data
|
|
if(outcome_family$family == "gaussian"){
|
|
sigma.y <- params[param.idx]
|
|
param.idx <- param.idx + 1
|
|
|
|
# outcome_formula likelihood using linear regression
|
|
ll.y.obs <- dnorm(y.obs, outcome.params %*% t(outcome.model.matrix),sd=sigma.y, log=TRUE)
|
|
}
|
|
|
|
df.obs <- model.frame(proxy_formula,df)
|
|
n.proxy.model.covars <- dim(proxy.model.matrix)[2]
|
|
proxy.params <- params[param.idx:(n.proxy.model.covars+param.idx-1)]
|
|
param.idx <- param.idx + n.proxy.model.covars
|
|
proxy.obs <- with(df.obs, eval(parse(text=proxy.variable)))
|
|
|
|
if( (proxy_family$family=="binomial") & (proxy_family$link=='logit')){
|
|
ll.w.obs <- vector(mode='numeric',length=dim(proxy.model.matrix)[1])
|
|
|
|
# proxy_formula likelihood using logistic regression
|
|
ll.w.obs[proxy.obs==1] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==1,]),log=TRUE)
|
|
ll.w.obs[proxy.obs==0] <- plogis(proxy.params %*% t(proxy.model.matrix[proxy.obs==0,]),log=TRUE, lower.tail=FALSE)
|
|
}
|
|
|
|
df.obs <- model.frame(truth_formula, df)
|
|
truth.variable <- all.vars(truth_formula)[1]
|
|
truth.obs <- with(df.obs, eval(parse(text=truth.variable)))
|
|
truth.model.matrix <- model.matrix(truth_formula,df)
|
|
n.truth.model.covars <- dim(truth.model.matrix)[2]
|
|
|
|
truth.params <- params[param.idx:(n.truth.model.covars + param.idx - 1)]
|
|
|
|
if( (truth_family$family=="binomial") & (truth_family$link=='logit')){
|
|
ll.x.obs <- vector(mode='numeric',length=dim(truth.model.matrix)[1])
|
|
|
|
# truth_formula likelihood using logistic regression
|
|
ll.x.obs[truth.obs==1] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==1,]),log=TRUE)
|
|
ll.x.obs[truth.obs==0] <- plogis(truth.params %*% t(truth.model.matrix[truth.obs==0,]),log=TRUE, lower.tail=FALSE)
|
|
}
|
|
|
|
# add the three likelihoods
|
|
ll.obs <- sum(ll.y.obs + ll.w.obs + ll.x.obs)
|
|
|
|
## likelihood for the predicted data
|
|
## integrate out the "truth" variable.
|
|
|
|
if(truth_family$family=='binomial'){
|
|
df.unobs <- df[is.na(eval(parse(text=truth.variable)))]
|
|
df.unobs.x1 <- copy(df.unobs)
|
|
df.unobs.x1[,'x'] <- 1
|
|
df.unobs.x0 <- copy(df.unobs)
|
|
df.unobs.x0[,'x'] <- 0
|
|
outcome.unobs <- with(df.unobs, eval(parse(text=response.var)))
|
|
|
|
outcome.model.matrix.x0 <- model.matrix(outcome_formula, df.unobs.x0)
|
|
outcome.model.matrix.x1 <- model.matrix(outcome_formula, df.unobs.x1)
|
|
if(outcome_family$family=="gaussian"){
|
|
|
|
# likelihood of outcome
|
|
ll.y.x0 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x0), sd=sigma.y, log=TRUE)
|
|
ll.y.x1 <- dnorm(outcome.unobs, outcome.params %*% t(outcome.model.matrix.x1), sd=sigma.y, log=TRUE)
|
|
}
|
|
|
|
if( (proxy_family$family=='binomial') & (proxy_family$link=='logit')){
|
|
|
|
proxy.model.matrix.x0 <- model.matrix(proxy_formula, df.unobs.x0)
|
|
proxy.model.matrix.x1 <- model.matrix(proxy_formula, df.unobs.x1)
|
|
proxy.unobs <- df.unobs[[proxy.variable]]
|
|
ll.w.x0 <- vector(mode='numeric', length=dim(df.unobs)[1])
|
|
ll.w.x1 <- vector(mode='numeric', length=dim(df.unobs)[1])
|
|
|
|
# likelihood of proxy
|
|
ll.w.x0[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==1,]), log=TRUE)
|
|
ll.w.x1[proxy.unobs==1] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==1,]), log=TRUE)
|
|
|
|
ll.w.x0[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x0[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
|
|
ll.w.x1[proxy.unobs==0] <- plogis(proxy.params %*% t(proxy.model.matrix.x1[proxy.unobs==0,]), log=TRUE,lower.tail=FALSE)
|
|
}
|
|
|
|
if(truth_family$link=='logit'){
|
|
truth.model.matrix <- model.matrix(truth_formula, df.unobs.x0)
|
|
# likelihood of truth
|
|
ll.x.x1 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE)
|
|
ll.x.x0 <- plogis(truth.params %*% t(truth.model.matrix), log=TRUE, lower.tail=FALSE)
|
|
}
|
|
}
|
|
|
|
ll.x0 <- ll.y.x0 + ll.w.x0 + ll.x.x0
|
|
ll.x1 <- ll.y.x1 + ll.w.x1 + ll.x.x1
|
|
ll.unobs <- sum(colLogSumExps(rbind(ll.x0, ll.x1)))
|
|
return(-(ll.unobs + ll.obs))
|
|
}
|
|
|
|
outcome.params <- colnames(model.matrix(outcome_formula,df))
|
|
lower <- rep(-Inf, length(outcome.params))
|
|
|
|
if(outcome_family$family=='gaussian'){
|
|
params <- c(outcome.params, 'sigma_y')
|
|
lower <- c(lower, 0.00001)
|
|
} else {
|
|
params <- outcome.params
|
|
}
|
|
|
|
proxy.params <- colnames(model.matrix(proxy_formula, df))
|
|
params <- c(params, paste0('proxy_',proxy.params))
|
|
lower <- c(lower, rep(-Inf, length(proxy.params)))
|
|
|
|
truth.params <- colnames(model.matrix(truth_formula, df))
|
|
params <- c(params, paste0('truth_', truth.params))
|
|
lower <- c(lower, rep(-Inf, length(truth.params)))
|
|
start <- rep(0.1,length(params))
|
|
names(start) <- params
|
|
|
|
if(method=='optim'){
|
|
fit <- optim(start, fn = measerr_mle_nll, lower=lower, method='L-BFGS-B', hessian=TRUE, control=list(maxit=1e6))
|
|
} else { # method='mle2'
|
|
|
|
quoted.names <- gsub("[\\(\\)]",'',names(start))
|
|
|
|
text <- paste("function(", paste0(quoted.names,'=',start,collapse=','),"){params<-c(",paste0(quoted.names,collapse=','),");return(measerr_mle_nll(params))}")
|
|
|
|
measerr_mle_nll_mle <- eval(parse(text=text))
|
|
names(start) <- quoted.names
|
|
names(lower) <- quoted.names
|
|
fit <- mle2(minuslogl=measerr_mle_nll_mle, start=start, lower=lower, parnames=params,control=list(maxit=1e6),method='L-BFGS-B')
|
|
}
|
|
|
|
return(fit)
|
|
}
|
|
|