library(ROCR)
source('classification-v1-2.r')
res=test.credal(300,1)


source('classification-v1-3.r')
res=test.credal(300,1,c(1,5),k=3) ## k=10, slower but same idea
colMeans(res)


source('classification-v2.r')
mammo = getdata()
set.seed(1)
res=kfcv.classifier(mammo, 1:5, 6, classifier.credal(1e-12,FALSE),k=10) ##MAR
colMeans(res,na.rm=T)
set.seed(1)
res2=kfcv.classifier(mammo, 1:5, 6, classifier.credal(1e-12,TRUE),k=10) ## CUR
colMeans(res2,na.rm=T)
 
mammo = getdata()
mammo[intersect(1:100,which(mammo[,5]==1)),5] = NA
mammo=mammo[1:200,]




##CAN PROB TELL IF WE WILL GUESS RIGHT OR WRONG?
pred <- prediction(res$robust,res$acc)
perf = performance(pred, measure = "acc")
plot(perf)
perf = performance(pred, measure = "ppv")
plot(perf)
##x=rev(unlist(perf@x.values)[-1])
##y=rev(unlist(perf@y.values)[-1])
##plot(stepfun(x,c(y,y[length(y)]),right=T))

## ACCURACY ABOVE VS ACCURACY BELOW (weird plot)
##perf <- performance(pred, measure = "ppv", x.measure = "pcmiss")
##plot(perf, colorize=T, #print.cutoffs.at=perf@alpha.values,
##      lwd=3,
##       text.adj=c(1.2,1.2))


pred.credal <- prediction(res$robust.1,res$acc)
perf = performance(pred.credal, measure = "acc")
plot(perf)
perf = performance(pred.credal, measure = "ppv")
plot(perf)
perf <- performance(pred.credal, measure = "ppv", x.measure = "pcmiss")
plot(perf, colorize=T, #print.cutoffs.at=perf@alpha.values,
       lwd=3,
       text.adj=c(1.2,1.2))



plot.c = function(res,xb=0,show.incorrect=F) {
pred <- prediction(res$robust.1,res$acc)
o=order(unlist(pred@predictions),decreasing=T)
plot(unlist(pred@predictions)[o]/max(unlist(pred@predictions)),cumsum(rep_len(1,length(unlist(pred@labels)[o]))),type='l',
col=2,xlim=c(xb,1),xlab="robustness measure (red) / prob (black)",
ylab="cases (cont.) / correct classified (dashed) above threshold")
pred <- prediction(res$robust,res$acc)
o=order(unlist(pred@predictions),decreasing=T)
points(unlist(pred@predictions)[o], cumsum(rep_len(1,length(unlist(pred@labels)[o]))),type='l',col=1)

pred <- prediction(res$robust.1,res$acc)
o=order(unlist(pred@predictions),decreasing=T)
points(unlist(pred@predictions)[o]/max(unlist(pred@predictions)),cumsum(as.numeric(unlist(pred@labels)[o])-1),type='l',col=2,lty=2)
pred <- prediction(res$robust,res$acc)
o=order(unlist(pred@predictions),decreasing=T)
points(unlist(pred@predictions)[o],cumsum(as.numeric(unlist(pred@labels)[o])-1),type='l',col=1,lty=2)

if(show.incorrect) {
pred <- prediction(res$robust.1,res$acc)
o=order(unlist(pred@predictions),decreasing=T)
points(unlist(pred@predictions)[o]/max(unlist(pred@predictions)),cumsum(2-as.numeric(unlist(pred@labels)[o])),type='l', col=2,xlim=c(xb,1),lty=3)
pred <- prediction(res$robust,res$acc)
o=order(unlist(pred@predictions),decreasing=T)
points(unlist(pred@predictions)[o],cumsum(2-as.numeric(unlist(pred@labels)[o])),type='l',col=1,lty=3)
}
}
