Minor changes

35 files changed, 157 insertions, 139 deletions

diff --git a/analysis/days.R b/analysis/days.R
index 2155819..4997ffd 100644
--- a/analysis/days.R
+++ b/analysis/days.R
@@ -30,12 +30,13 @@ nwireless=2
 nsimkey=4
 nsimulations=nseed*nwakeupfor*nwireless*nsimkey # Must be 3200
 
-## Load data
+## Load data and prepare the data
 data=suppressMessages(read_csv("../CCGRID2022.csv"))%>%distinct() # Note that in the data experiment wireless=="lora",seed==1,wakeupfor==60,simkey=="baseline" is present 2 times in the CSV file
 data_seed=data%>%group_by(simkey,wireless,wakeupfor,seed)%>%summarize(energy=sum(energy),coverage=sum(nDataRcv))%>%
     mutate(efficiency=energy/coverage)%>%
     ungroup()
 
+## F1_Score of multiclass vectors
 F1_Score2=function(truth, pred){
     result=sapply(c("baseline","extended","hint","hintandextended"),function(c){
         cur_truth=truth[truth==c]
@@ -48,6 +49,7 @@ F1_Score2=function(truth, pred){
     do.call("cbind",result)
 }
 
+## Train models
 generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl="lora",wuf=180) {
     attempts=seq(1,attempts_max)
     results=sapply(attempts,function(attempt){
@@ -94,98 +96,66 @@ generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl
 
 }
 
-generate_accuracy_energy = function(wireless,wakeupfor,steps=10, accuracy=10,ignore_hint=TRUE){
+generate_accuracy_energy = function(wireless,wakeupfor,steps=1, accuracy=10,ignore_hint=TRUE){
+    ## Setup variables
     npolicies=4
     data_seed_for_energy=data_seed%>%ungroup()
     if(ignore_hint){npolicies=npolicies-1;data_seed_for_energy=data_seed%>%filter(simkey!="hint")}
     data_seed_for_energy=data_seed_for_energy%>%filter(wireless==!!wireless,wakeupfor==!!wakeupfor)
+
     ## Generate inputs
     result=tibble()
     result_energy=tibble()
     for(i in seq(1,160,by=steps)){ # We stop at 80% of the data (this way test set is at least 20%)
+        print(paste("Step",i))
         acc=generate_accuracy_for(ignore_hint=ignore_hint,seed=i,attempts_max=accuracy,wrl=wireless,wuf=wakeupfor)
-        result_energy=rbind(result_energy,data_seed_for_energy%>%filter(seed<=i)%>%summarize(energy=sum(energy),seed_max=i,days=i*npolicies,months=days/30,setup=paste0(!!wireless," ",!!wakeupfor,"s"),wireless=!!wireless,wakeupfor=!!wakeupfor))
+        result_energy=rbind(result_energy,data_seed_for_energy%>%filter(seed<=i)%>%summarize(energy=sum(energy),seed_max=i,days=i*npolicies,setup=paste0(!!wireless," ",!!wakeupfor,"s"),wireless=!!wireless,wakeupfor=!!wakeupfor))
         result=rbind(result,acc)
     }
-   list(accuracy=result%>%mutate(days=seed_max*npolicies,months=days/30), # Since 3 policies (since ignore_hint=TRUE)
+   list(accuracy=result%>%mutate(days=seed_max*npolicies), # Since 3 policies (since ignore_hint=TRUE)
         energy=result_energy)
 }
 
-result_energy_policy=rbind(data_seed,
-                           data_seed%>%mutate(seed=seed+200),
-                           data_seed%>%mutate(seed=seed+400),
-                           data_seed%>%mutate(seed=seed+600)) # Almost same as if each experiment run 4 times more seed seed
-result_energy_policy=result_energy_policy%>%group_by(wireless,wakeupfor,simkey)%>%mutate(energy=cumsum(energy),setup=paste0(wireless," ",wakeupfor,"s"),days=seed,months=days/30)
 
-# Generate accuracy for each wireless and uptime
+########## Generate accuracy, energy, F1-Score and coverage data ##########
 if(F){ # Toggle to train
     lora60=generate_accuracy_energy("lora",60)
     lora180=generate_accuracy_energy("lora",180)
     nbiot60=generate_accuracy_energy("nbiot",60)
     nbiot180=generate_accuracy_energy("nbiot",180)
     accuracy=rbind(lora60$accuracy,lora180$accuracy,nbiot60$accuracy,nbiot180$accuracy)
+    coverage=data_seed%>%filter(simkey!="hint")%>%group_by(wireless,wakeupfor,seed)%>%summarize(coverage=sum(coverage))%>%mutate(days=seed*3)%>%filter(days %in% !!lora60$energy$days)%>%select(-seed)
     energy=rbind(lora60$energy,
                  lora180$energy,
                  nbiot60$energy,
-                 nbiot180$energy)
-#    energy$setup=factor(energy$setup,levels=c("lora 60s","nbiot 60s","lora 180s", "nbiot 180s"),ordered=TRUE)
+                 nbiot180$energy)%>%left_join(coverage,by=c("wireless","wakeupfor","days"))      
 }
-## Summarize
-result_summary=accuracy%>%group_by(wireless,wakeupfor,months,model)%>%
-    summarize(
-        mean_accuracy=mean(accuracy),sd_accuracy=sd(accuracy),min_accuracy=min(accuracy),max_accuracy=max(accuracy),
-        mean_f1_baseline=mean(f1_baseline),sd_f1_baseline=sd(f1_baseline),min_f1_baseline=min(f1_baseline),max_f1_baseline=max(f1_baseline),
-        mean_f1_hint=mean(f1_hint),sd_f1_hint=sd(f1_hint),min_f1_hint=min(f1_hint),max_f1_hint=max(f1_hint),
-        mean_f1_extended=mean(f1_extended),sd_f1_extended=sd(f1_extended),min_f1_extended=min(f1_extended),max_f1_extended=max(f1_extended),
-        mean_f1_hintandextended=mean(f1_hintandextended),sd_f1_hintandextended=sd(f1_hintandextended),min_f1_hintandextended=min(f1_hintandextended),max_f1_hintandextended=max(f1_hintandextended))
-
-## Result max
-metrics_peak=result_summary%>%group_by(wireless,wakeupfor,model)%>%
-    summarize(max_accuracy=max(mean_accuracy))
-
-
-
-
-
-
-result_energy_policy=result_energy_policy%>%filter(days %in% !!energy$days)
-csv_table=result_energy_policy%>%
-    full_join(energy,by=c("days","wireless","wakeupfor"),suffix=c("","_training"))
-csv_table=csv_table%>%group_by(wireless,wakeupfor)%>%summarize(delta=energy-energy_training,simkey=simkey,days=days,months=months)
-
-
-write("wireless,wakeupfor,policy,slope,intercept","figures/delta_energy.csv")
-csv_table%>%group_by(wireless,wakeupfor,simkey)%>%group_walk(function(data,grp){
-    grp=as.list(grp)
-    reg=lm(delta/1e3 ~ months,data)
-    slope=round(as.numeric(reg$coefficients["months"]),digits=1)
-    intercept=round(as.numeric(reg$coefficients[1]),digits=1)
-    print(paste0("Wireless=",grp$wireless," Wakeupfor=",grp$wakeupfor," Policy=",grp$simkey," Slope=",slope," Intercept=",intercept))
-    write(paste(grp$wireless,grp$wakeupfor,grp$simkey,slope,intercept,sep=","),"figures/delta_energy.csv",append=T)
-})
-
-
-ggplot(csv_table,aes(months,delta/1e3,color=simkey,shape=simkey))+
-    geom_line(size=1.2)+geom_point(size=3)+ylab("Delta in energy (kJ)")+xlab("Training months")+
-    facet_wrap(~wireless+wakeupfor,scale="free")+
-    scale_color_viridis(discrete=TRUE,option="H",end=0.95)
-
-stopifnot(1)
-
-ggplot(data=energy,aes(months,energy/1e6,group=setup,fill=setup))+
-    geom_bar(stat="identity",position="dodge")+
-    scale_fill_viridis(discrete=TRUE,option="D")+
-    labs(fill="Nodes wireless technology and uptime")+theme(legend.position=c(0.2,0.75))+
-    scale_x_continuous(breaks = seq(0, 15, by = 1))+
-    xlab("Number of months")+ylab("Energy consumption (MJ)")+
-    geom_point(data=result_energy_policy%>%filter(simkey=="hintandextended"),aes(months,energy/1e6,group=setup,color=simkey),size=1)+facet_wrap(~wireless+wakeupfor)
-ggsave("figures/months_energy.pdf",width=8.5,height=4)
 
 
-ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(linetype=model))+
-    geom_line(aes(months,mean_f1_baseline,color="Baseline"),size=1.2)+
-    geom_line(aes(months,mean_f1_extended,color="Extended"),size=1.2)+
-    geom_line(aes(months,mean_f1_hintandextended,color="Hintandextended"),size=1.2)+labs(color="Classes",linetype="Model")+
+########## Build learning curve (Accuracy+F1_Scores) ##########
+learning_curves=accuracy%>%group_by(wireless,wakeupfor,days,model)%>%
+    summarize(
+        ## Accuracy
+        mean_accuracy=mean(accuracy),sd_accuracy=sd(accuracy),
+        min_accuracy=min(accuracy),max_accuracy=max(accuracy),
+        ## F1-Score Baseline
+        mean_f1_baseline=mean(f1_baseline),sd_f1_baseline=sd(f1_baseline),
+        min_f1_baseline=min(f1_baseline),max_f1_baseline=max(f1_baseline),
+        ## F1-Score Hint
+        mean_f1_hint=mean(f1_hint),sd_f1_hint=sd(f1_hint),min_f1_hint=min(f1_hint),
+        max_f1_hint=max(f1_hint),
+        ## F1-Score Extended
+        mean_f1_extended=mean(f1_extended),sd_f1_extended=sd(f1_extended),
+        min_f1_extended=min(f1_extended),max_f1_extended=max(f1_extended),
+        ## F1-Score HintAndExtended
+        mean_f1_hintandextended=mean(f1_hintandextended),sd_f1_hintandextended=sd(f1_hintandextended),
+        min_f1_hintandextended=min(f1_hintandextended),max_f1_hintandextended=max(f1_hintandextended))
+
+
+ggplot(data=learning_curves%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(linetype=model))+
+    geom_line(aes(days,mean_f1_baseline,color="Baseline"),size=1.2)+
+    geom_line(aes(days,mean_f1_extended,color="Extended"),size=1.2)+
+    geom_line(aes(days,mean_f1_hintandextended,color="Hintandextended"),size=1.2)+labs(color="Classes",linetype="Model")+
     facet_wrap(~wireless+wakeupfor)+
     scale_x_continuous(breaks = seq(0, 15, by = 1))+
     scale_y_continuous(breaks = seq(0, 1, by = 0.1))+
@@ -196,14 +166,14 @@ ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(l
           legend.spacing=unit(-0.2,"cm"),
           legend.box.margin=margin(1,1,1,1),
           legend.box.background = element_rect(fill = "white", color = "black",size=0.8))+
-    xlab("Training months")+ylab("Classes F1-Score")+
+    xlab("Training days")+ylab("Classes F1-Score")+
     scale_color_viridis(discrete=TRUE,end=0.7)
-ggsave("figures/months_f1-score.pdf",width=8.5,height=6)
+ggsave("figures/days_f1-score.pdf",width=8.5,height=6)
 
 
 ## Plot Merge Accuracy
-ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(months,mean_accuracy))+
-    geom_line(aes(linetype=model),size=1.2)+xlab("Training months")+ylab("Model accuracy")+labs(linetype="Models")+
+ggplot(data=learning_curves%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(days,mean_accuracy))+
+    geom_line(aes(linetype=model),size=1.2)+xlab("Training days")+ylab("Model accuracy")+labs(linetype="Models")+
     scale_x_continuous(breaks = seq(0, 15, by = 1))+
     scale_y_continuous(breaks = seq(0, 1, by = 0.1))+
     facet_wrap(~wireless+wakeupfor)+
@@ -212,29 +182,57 @@ ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(m
           legend.position = c(0.38,0.68),
           legend.background = element_rect(fill = "white", color = "black",size=0.8))+
     scale_color_viridis(discrete=TRUE,end=0.7)
-ggsave("figures/months_accuracy.pdf",width=8.5,height=6)
-stopifnot(1)
-
-## Plot accuracy + F1-Score
-sapply(c("knn","tree"),function(grp){
-    data=result_summary%>%filter(model==grp)
-    plot=ggplot(data,aes(months,mean_accuracy))+
-        geom_ribbon(aes(ymin=mean_accuracy-sd_accuracy,ymax=mean_accuracy+sd_accuracy),alpha=0.2,color=NA)+
-        geom_line(size=1.1)+geom_point(size=3)+
-        geom_point(data=data%>%drop_na(mean_f1_baseline),aes(months,mean_f1_baseline,color="baseline"))+geom_line(data=data%>%drop_na(mean_f1_baseline),aes(months,mean_f1_baseline,color="baseline"))+
-        geom_point(data=data%>%drop_na(mean_f1_extended),aes(months,mean_f1_extended,color="extended"))+geom_line(data=data%>%drop_na(mean_f1_extended),aes(months,mean_f1_extended,color="extended"))+
-        geom_point(data=data%>%drop_na(mean_f1_hintandextended),aes(months,mean_f1_hintandextended,color="hintandextended"))+geom_line(data=data%>%drop_na(mean_f1_hintandextended),aes(months,mean_f1_hintandextended,color="hintandextended"))
-
-    if(any(!is.na(data$mean_f1_hint))){
-        plot=plot+geom_point(data=data%>%drop_na(mean_f1_hint),aes(months,mean_f1_hint,color="hint"))+geom_line(data=data%>%drop_na(mean_f1_hint),aes(months,mean_f1_hint,color="hint"))
-    }
+ggsave("figures/days_accuracy.pdf",width=8.5,height=6)
+
+
+########## Energy and delta with raw policies ##########
+## First we extended the number of seed to cover the entire duration of the training
+data_seed_energy=rbind(data_seed,
+                           data_seed%>%mutate(seed=seed+200),
+                           data_seed%>%mutate(seed=seed+400),
+                           data_seed%>%mutate(seed=seed+600)) # Almost same as if each experiment run 4 times more seed seed
+## Compute the cumulative energy of each policies in each configuration accross the 4*200 days
+data_seed_energy=data_seed_energy%>%group_by(wireless,wakeupfor,simkey)%>%mutate(energy=cumsum(energy),setup=paste0(wireless," ",wakeupfor,"s"),days=seed)
+## Now filter the data
+data_seed_energy=data_seed_energy%>%filter(days %in% !!energy$days)
+## Compute the delta data
+energy_coverage_delta=data_seed_energy%>%
+    full_join(energy,by=c("days","wireless","wakeupfor"),suffix=c("","_training"))
+energy_coverage_delta=energy_coverage_delta%>%group_by(wireless,wakeupfor)%>%summarize(delta_energy=energy-energy_training,simkey=simkey,days=days,delta_coverage=coverage-coverage_training)
+
+
+
 
-    plot=plot+xlab("Number of months")+ylab(paste("Mean",grp,"accuracy"))+ggtitle(paste(grp,"accuracy"))+
-#        geom_hline(data=metrics_peak%>%filter(model==grp),aes(yintercept=max_accuracy),color="red",size=1)+
-#        geom_text(data=metrics_peak%>%filter(model==grp),x=0,aes(y=max_accuracy,label = round(max_accuracy,digits=1),vjust=-1),color="red")+
-        facet_wrap(~wireless+wakeupfor)+scale_x_continuous(breaks = seq(0, max(result_summary$months), by = 1))+ylim(c(NA,1))+labs(color="F1 Score")+
-    theme(legend.position="top")
-    ggsave(paste0("figures/months_",grp,".pdf"),width=9,height=8)
-    print(plot)
+write("wireless,wakeupfor,policy,slope,intercept,delta_coverage","figures/delta_energy_coverage.csv")
+energy_coverage_delta%>%group_by(wireless,wakeupfor,simkey)%>%group_walk(function(data,grp){
+    grp=as.list(grp)
+    reg=lm(delta_energy/1e3 ~ days,data)
+    slope=round(as.numeric(reg$coefficients["days"]),digits=1)
+    intercept=round(as.numeric(reg$coefficients[1]),digits=1)
+    mean_delta_coverage=round(mean(data$delta_coverage),digits=1)
+    print(paste0("Wireless=",grp$wireless," Wakeupfor=",grp$wakeupfor," Policy=",grp$simkey," Slope=",slope," Intercept=",intercept," Delta Coverage=",mean_delta_coverage))
+    write(paste(grp$wireless,grp$wakeupfor,grp$simkey,slope,intercept,mean_delta_coverage,sep=","),"figures/delta_energy_coverage.csv",append=T)
 })
 
+ggplot(energy_coverage_delta,aes(days,delta_energy/1e3,color=simkey,shape=simkey))+
+    geom_line(size=1.2)+ylab("Delta in energy (kJ)")+xlab("Training days")+
+    facet_wrap(~wireless+wakeupfor,scale="free")+
+    scale_color_viridis(discrete=TRUE,option="H",end=0.95)
+ggsave("figures/delta_energy_training.pdf")
+
+ggplot(energy_coverage_delta,aes(days,delta_coverage,color=simkey))+
+    geom_line(size=1.2)+ylab("Delta in coverage")+xlab("Training days")+
+    facet_wrap(~wireless+wakeupfor,scale="free")+
+    scale_color_viridis(discrete=TRUE,end=0.9)
+ggsave("figures/delta_coverage_training.pdf")
+
+ggplot(data=energy%>%filter(((days/3) %% 6) == 0),aes(days,energy/1e6,group=setup,fill=setup))+
+    geom_bar(stat="identity",position="dodge")+
+    scale_fill_viridis(discrete=TRUE,option="D")+
+    labs(fill="Nodes wireless technology and uptime")+theme(legend.position=c(0.2,0.75))+
+    scale_x_continuous(breaks = seq(0, max(energy$days), by = 15))+
+    xlab("Number of days")+ylab("Energy consumption (MJ)")
+ggsave("figures/days_energy.pdf",width=8.5,height=4)
+
+
+
diff --git a/analysis/figures/combined.pdf b/analysis/figures/combined.pdf
deleted file mode 100644
index 68e9bd3..0000000
--- a/analysis/figures/combined.pdf
+++ /dev/null
diff --git a/analysis/figures/days_accuracy.pdf b/analysis/figures/days_accuracy.pdf
new file mode 100644
index 0000000..e97946c
--- /dev/null
+++ b/analysis/figures/days_accuracy.pdf
diff --git a/analysis/figures/knn_elbow_NoHintIsFALSE.pdf b/analysis/figures/days_energy.pdf
index 2499924..1618261 100644
--- a/analysis/figures/knn_elbow_NoHintIsFALSE.pdf
+++ b/analysis/figures/days_energy.pdf
diff --git a/analysis/figures/days_f1-score.pdf b/analysis/figures/days_f1-score.pdf
new file mode 100644
index 0000000..bdd9bfa
--- /dev/null
+++ b/analysis/figures/days_f1-score.pdf
diff --git a/analysis/figures/delta_coverage_training.pdf b/analysis/figures/delta_coverage_training.pdf
new file mode 100644
index 0000000..a1b82e1
--- /dev/null
+++ b/analysis/figures/delta_coverage_training.pdf
diff --git a/analysis/figures/delta_energy.csv b/analysis/figures/delta_energy.csv
index 3b622ef..78ecd38 100644
--- a/analysis/figures/delta_energy.csv
+++ b/analysis/figures/delta_energy.csv
@@ -1,17 +1,49 @@
-wireless,wakeupfor,policy,slope,intercept
-lora,60,baseline,-16.3,-0.7
-lora,60,extended,6.4,-0.5
-lora,60,hint,-14.2,-0.5
-lora,60,hintandextended,10.1,-0.6
-lora,180,baseline,-20.3,0.1
-lora,180,extended,-14.5,0.1
-lora,180,hint,93.9,-2.3
-lora,180,hintandextended,35,0.7
-nbiot,60,baseline,-2.9,-2.4
-nbiot,60,extended,-2.4,-2
-nbiot,60,hint,6.5,-1.5
-nbiot,60,hintandextended,5.3,0.3
-nbiot,180,baseline,-31.6,2.8
-nbiot,180,extended,-36.2,2.7
-nbiot,180,hint,78.1,2.2
-nbiot,180,hintandextended,67.5,0.9
+wireless,wakeupfor,policy,slope,intercept,delta_coverage
+lora,60,baseline,-0.5,-1.1,-12.6
+lora,60,extended,0.2,-0.9,-6.4
+lora,60,hint,-0.5,-0.8,-12.6
+lora,60,hintandextended,0.3,-0.9,-5.9
+lora,180,baseline,-0.7,0,-22.7
+lora,180,extended,-0.5,0.1,-14.1
+lora,180,hint,3.1,-3.3,-13.8
+lora,180,hintandextended,1.2,0.7,-13
+nbiot,60,baseline,-0.1,-2.8,-13.9
+nbiot,60,extended,-0.1,-2.6,-10
+nbiot,60,hint,0.2,-1.6,-11.5
+nbiot,60,hintandextended,0.2,0,-8.8
+nbiot,180,baseline,-1.1,3.1,-23
+nbiot,180,extended,-1.2,3,-22.3
+nbiot,180,hint,2.6,2.4,-21.6
+nbiot,180,hintandextended,2.2,1.1,-21.5
+lora,60,baseline,-0.5,-1.1,-12.6
+lora,60,extended,0.2,-0.9,-6.4
+lora,60,hint,-0.5,-0.8,-12.6
+lora,60,hintandextended,0.3,-0.9,-5.9
+lora,180,baseline,-0.7,0,-22.7
+lora,180,extended,-0.5,0.1,-14.1
+lora,180,hint,3.1,-3.3,-13.8
+lora,180,hintandextended,1.2,0.7,-13
+nbiot,60,baseline,-0.1,-2.8,-13.9
+nbiot,60,extended,-0.1,-2.6,-10
+nbiot,60,hint,0.2,-1.6,-11.5
+nbiot,60,hintandextended,0.2,0,-8.8
+nbiot,180,baseline,-1.1,3.1,-23
+nbiot,180,extended,-1.2,3,-22.3
+nbiot,180,hint,2.6,2.4,-21.6
+nbiot,180,hintandextended,2.2,1.1,-21.5
+lora,60,baseline,-0.5,-1.1,-12.6
+lora,60,extended,0.2,-0.9,-6.4
+lora,60,hint,-0.5,-0.8,-12.6
+lora,60,hintandextended,0.3,-0.9,-5.9
+lora,180,baseline,-0.7,0,-22.7
+lora,180,extended,-0.5,0.1,-14.1
+lora,180,hint,3.1,-3.3,-13.8
+lora,180,hintandextended,1.2,0.7,-13
+nbiot,60,baseline,-0.1,-2.8,-13.9
+nbiot,60,extended,-0.1,-2.6,-10
+nbiot,60,hint,0.2,-1.6,-11.5
+nbiot,60,hintandextended,0.2,0,-8.8
+nbiot,180,baseline,-1.1,3.1,-23
+nbiot,180,extended,-1.2,3,-22.3
+nbiot,180,hint,2.6,2.4,-21.6
+nbiot,180,hintandextended,2.2,1.1,-21.5
diff --git a/analysis/figures/delta_energy_coverage.csv b/analysis/figures/delta_energy_coverage.csv
new file mode 100644
index 0000000..60d139e
--- /dev/null
+++ b/analysis/figures/delta_energy_coverage.csv
@@ -0,0 +1,17 @@
+wireless,wakeupfor,policy,slope,intercept,delta_coverage
+lora,60,baseline,-0.5,-1.1,-12.6
+lora,60,extended,0.2,-0.9,-6.4
+lora,60,hint,-0.5,-0.8,-12.6
+lora,60,hintandextended,0.3,-0.9,-5.9
+lora,180,baseline,-0.7,0,-22.7
+lora,180,extended,-0.5,0.1,-14.1
+lora,180,hint,3.1,-3.3,-13.8
+lora,180,hintandextended,1.2,0.7,-13
+nbiot,60,baseline,-0.1,-2.8,-13.9
+nbiot,60,extended,-0.1,-2.6,-10
+nbiot,60,hint,0.2,-1.6,-11.5
+nbiot,60,hintandextended,0.2,0,-8.8
+nbiot,180,baseline,-1.1,3.1,-23
+nbiot,180,extended,-1.2,3,-22.3
+nbiot,180,hint,2.6,2.4,-21.6
+nbiot,180,hintandextended,2.2,1.1,-21.5
diff --git a/analysis/figures/delta_energy_training.pdf b/analysis/figures/delta_energy_training.pdf
new file mode 100644
index 0000000..eaa8e85
--- /dev/null
+++ b/analysis/figures/delta_energy_training.pdf
diff --git a/analysis/figures/dimension_coverage.pdf b/analysis/figures/dimension_coverage.pdf
deleted file mode 100644
index 4081087..0000000
--- a/analysis/figures/dimension_coverage.pdf
+++ /dev/null
diff --git a/analysis/figures/dimension_efficiency.pdf b/analysis/figures/dimension_efficiency.pdf
deleted file mode 100644
index bc639e4..0000000
--- a/analysis/figures/dimension_efficiency.pdf
+++ /dev/null
diff --git a/analysis/figures/dimension_energy-coverage-policy.pdf b/analysis/figures/dimension_energy-coverage-policy.pdf
deleted file mode 100644
index a0f62c2..0000000
--- a/analysis/figures/dimension_energy-coverage-policy.pdf
+++ /dev/null
diff --git a/analysis/figures/dimension_energy-coverage-wakeupfor.pdf b/analysis/figures/dimension_energy-coverage-wakeupfor.pdf
deleted file mode 100644
index 99607a5..0000000
--- a/analysis/figures/dimension_energy-coverage-wakeupfor.pdf
+++ /dev/null
diff --git a/analysis/figures/dimension_energy-coverage.pdf b/analysis/figures/dimension_energy-coverage.pdf
deleted file mode 100644
index 76d9306..0000000
--- a/analysis/figures/dimension_energy-coverage.pdf
+++ /dev/null
diff --git a/analysis/figures/dimension_energy.pdf b/analysis/figures/dimension_energy.pdf
deleted file mode 100644
index 5a5b51c..0000000
--- a/analysis/figures/dimension_energy.pdf
+++ /dev/null
diff --git a/analysis/figures/f1_scores_offline.csv b/analysis/figures/f1_scores_offline.csv
deleted file mode 100644
index fa754ff..0000000
--- a/analysis/figures/f1_scores_offline.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-model,f1_baseline,f1_hint,f1_extended,f1_hintandextended,accuracy
-knn,0.88,NA,0.89,0.91,0.81
-tree,0.93,NA,0.86,0.92,0.83
diff --git a/analysis/figures/f1_scores_offline_ignoreHINTFALSE.csv b/analysis/figures/f1_scores_offline_ignoreHINTFALSE.csv
deleted file mode 100644
index 6f8b376..0000000
--- a/analysis/figures/f1_scores_offline_ignoreHINTFALSE.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-model,f1_baseline,f1_hint,f1_extended,f1_hintandextended,accuracy
-knn,0.83,0.73,0.9,0.79,0.69
-tree,0.9,0.75,0.86,0.79,0.7
diff --git a/analysis/figures/f1_scores_offline_ignoreHINTTRUE.csv b/analysis/figures/f1_scores_offline_ignoreHINTTRUE.csv
deleted file mode 100644
index fa754ff..0000000
--- a/analysis/figures/f1_scores_offline_ignoreHINTTRUE.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-model,f1_baseline,f1_hint,f1_extended,f1_hintandextended,accuracy
-knn,0.88,NA,0.89,0.91,0.81
-tree,0.93,NA,0.86,0.92,0.83
diff --git a/analysis/figures/knn.csv b/analysis/figures/knn.csv
deleted file mode 100644
index 3b5cc10..0000000
--- a/analysis/figures/knn.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-f1_baseline
-0.905147752632288
diff --git a/analysis/figures/knn_elbow_NoHintIsTRUE.pdf b/analysis/figures/knn_elbow_NoHintIsTRUE.pdf
deleted file mode 100644
index 7737b06..0000000
--- a/analysis/figures/knn_elbow_NoHintIsTRUE.pdf
+++ /dev/null
diff --git a/analysis/figures/months_accuracy.pdf b/analysis/figures/months_accuracy.pdf
deleted file mode 100644
index 3bf5b94..0000000
--- a/analysis/figures/months_accuracy.pdf
+++ /dev/null
diff --git a/analysis/figures/months_energy.pdf b/analysis/figures/months_energy.pdf
deleted file mode 100644
index 7a5bb39..0000000
--- a/analysis/figures/months_energy.pdf
+++ /dev/null
diff --git a/analysis/figures/months_f1-score.pdf b/analysis/figures/months_f1-score.pdf
deleted file mode 100644
index 1880ffb..0000000
--- a/analysis/figures/months_f1-score.pdf
+++ /dev/null
diff --git a/analysis/figures/months_knn.pdf b/analysis/figures/months_knn.pdf
deleted file mode 100644
index 2eab11f..0000000
--- a/analysis/figures/months_knn.pdf
+++ /dev/null
diff --git a/analysis/figures/months_tree.pdf b/analysis/figures/months_tree.pdf
deleted file mode 100644
index 60fe389..0000000
--- a/analysis/figures/months_tree.pdf
+++ /dev/null
diff --git a/analysis/figures/random_inputs_NoHintIsFALSE.pdf b/analysis/figures/random_inputs_NoHintIsFALSE.pdf
deleted file mode 100644
index 450a430..0000000
--- a/analysis/figures/random_inputs_NoHintIsFALSE.pdf
+++ /dev/null
diff --git a/analysis/figures/random_inputs_NoHintIsTRUE.pdf b/analysis/figures/random_inputs_NoHintIsTRUE.pdf
deleted file mode 100644
index 61098e0..0000000
--- a/analysis/figures/random_inputs_NoHintIsTRUE.pdf
+++ /dev/null
diff --git a/analysis/figures/sim_dimension_coverage_NO_HINT.pdf b/analysis/figures/sim_dimension_coverage_NO_HINT.pdf
deleted file mode 100644
index 096e285..0000000
--- a/analysis/figures/sim_dimension_coverage_NO_HINT.pdf
+++ /dev/null
diff --git a/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf b/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf
deleted file mode 100644
index 567ec38..0000000
--- a/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf
+++ /dev/null
diff --git a/analysis/figures/sim_dimension_energy_NO_HINT.pdf b/analysis/figures/sim_dimension_energy_NO_HINT.pdf
deleted file mode 100644
index 7a8129f..0000000
--- a/analysis/figures/sim_dimension_energy_NO_HINT.pdf
+++ /dev/null
diff --git a/analysis/figures/sim_dimension_energy_WITH_HINT.pdf b/analysis/figures/sim_dimension_energy_WITH_HINT.pdf
deleted file mode 100644
index cfdbf83..0000000
--- a/analysis/figures/sim_dimension_energy_WITH_HINT.pdf
+++ /dev/null
diff --git a/analysis/figures/sim_results_NO_HINT.csv b/analysis/figures/sim_results_NO_HINT.csv
deleted file mode 100644
index bb766bd..0000000
--- a/analysis/figures/sim_results_NO_HINT.csv
+++ /dev/null
@@ -1,9 +0,0 @@
-wireless,wakeupfor,model,mean_energy,mean_coverage,sd_energy,sd_coverage
-lora,60,knn,-32.58,-0.06,98.03,0.72
-lora,60,tree,-155.83,-0.98,119.03,0.87
-lora,180,knn,-1125.21,5.26,99.11,0.22
-lora,180,tree,-1659.57,2.44,199.05,0.44
-nbiot,60,knn,-497.99,0.32,52.86,0.3
-nbiot,60,tree,-520.09,-0.09,60.35,0.34
-nbiot,180,knn,12.46,2.32,149.98,0.17
-nbiot,180,tree,-284.82,2.18,257.4,0.29
diff --git a/analysis/figures/sim_results_WITH_HINT.csv b/analysis/figures/sim_results_WITH_HINT.csv
deleted file mode 100644
index b029bef..0000000
--- a/analysis/figures/sim_results_WITH_HINT.csv
+++ /dev/null
@@ -1,9 +0,0 @@
-wireless,wakeupfor,model,mean_energy,mean_coverage,sd_energy,sd_coverage
-lora,60,knn,-171.89,-0.78,119.98,0.88
-lora,60,tree,-207.11,-1.05,123.19,0.9
-lora,180,knn,-2629.47,0.11,202.56,0.44
-lora,180,tree,-2924.29,-1.44,172.87,0.38
-nbiot,60,knn,-560.44,-0.53,67.91,0.38
-nbiot,60,tree,-521.77,0.19,61.78,0.35
-nbiot,180,knn,-1543.86,1.51,377.76,0.43
-nbiot,180,tree,-1874.18,1.36,357.03,0.41
diff --git a/analysis/figures/tree_FALSE.pdf b/analysis/figures/tree_FALSE.pdf
deleted file mode 100644
index 8ecbe5d..0000000
--- a/analysis/figures/tree_FALSE.pdf
+++ /dev/null
diff --git a/analysis/figures/tree_TRUE.pdf b/analysis/figures/tree_TRUE.pdf
deleted file mode 100644
index 95f9025..0000000
--- a/analysis/figures/tree_TRUE.pdf
+++ /dev/null