Update

16 files changed, 41 insertions, 37 deletions

diff --git a/analysis/days.R b/analysis/days.R
index 1ba131d..7d26aef 100644
--- a/analysis/days.R
+++ b/analysis/days.R
@@ -3,6 +3,7 @@ library("class")
 library("rpart")
 library("rpart.plot")
 library("viridis")
+library("MLmetrics")
 
 ## Simulation Parameters:
 ## simkey         {baseline,extended,hint,hintandextended}
@@ -40,8 +41,8 @@ generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl
     ## Prepare data for traning
     set.seed(1+attempt) # Reproducibility
     wireless_map=c("lora"=1,"nbiot"=2)
-    data_seed=data_seed%>%filter(wakeupfor==wuf,wireless==wrl)
-    data_ml=data_seed%>%select(-efficiency)%>%mutate(wireless=wireless_map[data_seed$wireless])
+    cur_data_seed=data_seed%>%filter(wakeupfor==wuf,wireless==wrl)
+    data_ml=cur_data_seed%>%select(-efficiency)%>%mutate(wireless=wireless_map[cur_data_seed$wireless])
     if(ignore_hint){
         data_ml=data_ml%>%filter(simkey!="hint")
     }
@@ -49,12 +50,15 @@ generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl
     test_set=data_ml%>%anti_join(train_set)%>%select(-seed)        # build test_sed excluding training set
 
     ## KNN training
-    knn_predictions=knn(train=train_set%>%select(-simkey),test=test_set%>%select(-simkey),cl=train_set$simkey,k=10)
+    knn_predictions=knn(train=train_set%>%select(-simkey),test=test_set%>%select(-simkey),cl=train_set$simkey,k=min(10,NROW(train_set)))
     ## KNN analysis
     knn_cont_table=table(knn_predictions,test_set$simkey)
     knn_accuracy=round((sum(diag(knn_cont_table)/sum(rowSums(knn_cont_table))))*100)
     knn_prop_table=round(prop.table(knn_cont_table),digits=2)
-
+    knn_f1_score=F1_Score(test_set$simkey,knn_predictions)
+    knn_recall=Recall(test_set$simkey,knn_predictions)
+    knn_precision=Precision(test_set$simkey,knn_predictions)
+        
     ## Decision tree
     tree=rpart(
         simkey ~ wireless + wakeupfor + energy + coverage,
@@ -66,7 +70,10 @@ generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl
     tree_cont_table=table(tree_predictions,test_set$simkey)
     tree_accuracy=round((sum(diag(tree_cont_table)/sum(rowSums(tree_cont_table))))*100)
     tree_prop_table=round(prop.table(tree_cont_table),digits=2)
-     list(tibble(seed_max=seed_max,knn_accuracy=knn_accuracy,tree_accuracy=tree_accuracy))
+    tree_f1_score=F1_Score(test_set$simkey,tree_predictions)
+    tree_recall=Recall(test_set$simkey,tree_predictions)
+    tree_precision=Precision(test_set$simkey,tree_predictions)
+     list(tibble(seed_max=seed_max,model=c("knn","tree"),accuracy=c(knn_accuracy,tree_accuracy),f1_score=c(knn_f1_score,tree_f1_score),recall=c(knn_recall,tree_recall),precision=c(knn_precision,tree_precision)))
     })
     ## Prints
     results=do.call("rbind",results)
@@ -74,52 +81,49 @@ generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl
 
 }
 
-generate_accuracy = function(wireless,wakeupfor,steps=20, accuracy=10){
+generate_accuracy = function(wireless,wakeupfor,steps=2, accuracy=20,ignore_hint=TRUE){
+    npolicies=4
+    if(ignore_hint){npolicies=npolicies-1}
     ## Generate inputs
     result=tibble()
-    for(i in seq(1,200,by=steps)){
-        acc=generate_accuracy_for(ignore_hint=TRUE,seed=i,attempts_max=accuracy,wrl=wireless,wuf=wakeupfor)
+    for(i in seq(1,160,by=steps)){ # We stop at 80% of the data (this way test set is at least 20%)
+        acc=generate_accuracy_for(ignore_hint=ignore_hint,seed=i,attempts_max=accuracy,wrl=wireless,wuf=wakeupfor)
         result=rbind(result,acc)
     }
-    result%>%mutate(days=seed_max*3) # Since 3 policies (since ignore_hint=TRUE)
+    result%>%mutate(days=seed_max*npolicies,months=days/30) # Since 3 policies (since ignore_hint=TRUE)
 }
 
-## Generate accuracy for each wireless and uptime
+# Generate accuracy for each wireless and uptime
 accuracy=rbind(generate_accuracy("lora",60),
                generate_accuracy("lora",180),
                generate_accuracy("nbiot",60),
                generate_accuracy("nbiot",180))
 
 ## Summarize
-result_summary=accuracy%>%group_by(wireless,wakeupfor,seed_max)%>%summarize(attempts_max=mean(attempts_max),days=mean(days),mean_knn_accuracy=mean(knn_accuracy),sd_knn_accuracy=sd(knn_accuracy),min_knn_accuracy=min(knn_accuracy),max_knn_accuracy=max(knn_accuracy),mean_tree_accuracy=mean(tree_accuracy),sd_tree_accuracy=sd(tree_accuracy),min_tree_accuracy=min(tree_accuracy),max_tree_accuracy=max(tree_accuracy))
+result_summary=accuracy%>%group_by(wireless,wakeupfor,months,model)%>%
+    summarize(
+        mean_accuracy=mean(accuracy),sd_accuracy=sd(accuracy),min_accuracy=min(accuracy),max_accuracy=max(accuracy),
+        mean_f1_score=mean(f1_score),sd_f1_score=sd(f1_score),min_f1_score=min(f1_score),max_f1_score=max(f1_score),
+        mean_recall=mean(recall),sd_recall=sd(recall),min_recall=min(recall),max_recall=max(recall),
+        mean_precision=mean(precision),sd_precision=sd(precision),min_precision=min(precision),max_precision=max(precision))
 
 ## Result max
-result_max=result_summary%>%group_by(wireless,wakeupfor)%>%summarize(max_knn_mean=max(mean_knn_accuracy),max_tree_mean=max(mean_tree_accuracy))
+metrics_peak=result_summary%>%group_by(wireless,wakeupfor,model)%>%
+    summarize(max_accuracy=max(mean_accuracy),
+              max_f1_score=max(mean_f1_score),
+              max_recall=max(mean_recall),
+              max_precision=max(mean_precision))
 
 ## Plot
-ggplot(result_summary,aes(days,mean_knn_accuracy))+
-    geom_errorbar(aes(ymin=min_knn_accuracy,ymax=max_knn_accuracy),width=5)+
-    geom_boxplot(aes(ymin=min_knn_accuracy,ymax=max_knn_accuracy,
-                                    middle=mean_knn_accuracy,
-                                    upper=mean_knn_accuracy+sd_knn_accuracy,
-                                    lower=mean_knn_accuracy-sd_knn_accuracy,group=days),stat="identity",fill="grey")+
-    geom_line(size=1.1)+geom_point(size=3,pch=15)+xlab("Number of training days")+ylab("Mean KNN accuracy")+ggtitle("KNN Accuracy")+
-    ylim(c(0,100))+
-    facet_wrap(~wireless+wakeupfor)+
-    geom_hline(data=result_max,aes(yintercept=max_knn_mean),color="red",size=1)+
-    geom_text(data=result_max, geom="text",x=0,aes(y=max_knn_mean,label = max_knn_mean,vjust=-1),color="red")
-ggsave("figures/days_knn.pdf")
-
-ggplot(result_summary,aes(days,mean_tree_accuracy))+
-    geom_errorbar(aes(ymin=min_tree_accuracy,ymax=max_tree_accuracy),width=5)+
-    geom_boxplot(aes(ymin=min_tree_accuracy,ymax=max_tree_accuracy,
-                                    middle=mean_tree_accuracy,
-                                    upper=mean_tree_accuracy+sd_tree_accuracy,
-                                    lower=mean_tree_accuracy-sd_tree_accuracy,group=days),stat="identity",fill="grey")+
-    geom_line(size=1.1)+geom_point(size=3,pch=15)+xlab("Number of training days")+ylab("Mean tree accuracy")+ggtitle("TREE Accuracy")+
-    ylim(c(0,100))+
-    facet_wrap(~wireless+wakeupfor)+
-    geom_hline(data=result_max,aes(yintercept=max_tree_mean),color="red",size=1)+
-    geom_text(data=result_max, geom="text",x=0,aes(y=max_tree_mean,label = max_tree_mean,vjust=-1),color="red")
-ggsave("figures/days_tree.pdf")
+sapply(c("knn","tree"),function(grp){
+    ggplot(result_summary%>%filter(model==grp),aes(months,mean_accuracy))+
+        geom_ribbon(aes(ymin=mean_accuracy-sd_accuracy,ymax=mean_accuracy+sd_accuracy),alpha=0.2,color=NA)+
+        geom_line(size=1.1)+geom_point(size=3)+xlab("Number of training months")+ylab(paste("Mean",grp,"accuracy"))+ggtitle(paste(grp,"accuracy"))+
+#        ylim(c(0,100))+
+        geom_hline(data=metrics_peak%>%filter(model==grp),aes(yintercept=max_accuracy),color="red",size=1)+
+        geom_text(data=metrics_peak%>%filter(model==grp),x=0,aes(y=max_accuracy,label = round(max_accuracy,digits=1),vjust=-1),color="red")+
+        facet_wrap(~wireless+wakeupfor)
+        scale_x_continuous(breaks = seq(0, max(result_summary$months), by = 1))
+    ggsave(paste0("figures/months_",grp,".pdf"))
+})
 
diff --git a/analysis/figures/combined.pdf b/analysis/figures/combined.pdf
index 11f3916..9ec8250 100644
--- a/analysis/figures/combined.pdf
+++ b/analysis/figures/combined.pdf
diff --git a/analysis/figures/days_knn.pdf b/analysis/figures/days_knn.pdf
deleted file mode 100644
index a3a4f88..0000000
--- a/analysis/figures/days_knn.pdf
+++ /dev/null
diff --git a/analysis/figures/days_tree.pdf b/analysis/figures/days_tree.pdf
deleted file mode 100644
index fb8c411..0000000
--- a/analysis/figures/days_tree.pdf
+++ /dev/null
diff --git a/analysis/figures/dimension_coverage.pdf b/analysis/figures/dimension_coverage.pdf
index c7575b6..7d6e543 100644
--- a/analysis/figures/dimension_coverage.pdf
+++ b/analysis/figures/dimension_coverage.pdf
diff --git a/analysis/figures/dimension_efficiency.pdf b/analysis/figures/dimension_efficiency.pdf
index 59c4f38..6638a1e 100644
--- a/analysis/figures/dimension_efficiency.pdf
+++ b/analysis/figures/dimension_efficiency.pdf
diff --git a/analysis/figures/dimension_energy-coverage-policy.pdf b/analysis/figures/dimension_energy-coverage-policy.pdf
index 0281894..90a05bd 100644
--- a/analysis/figures/dimension_energy-coverage-policy.pdf
+++ b/analysis/figures/dimension_energy-coverage-policy.pdf
diff --git a/analysis/figures/dimension_energy-coverage-wakeupfor.pdf b/analysis/figures/dimension_energy-coverage-wakeupfor.pdf
index 9f51c55..86a0e52 100644
--- a/analysis/figures/dimension_energy-coverage-wakeupfor.pdf
+++ b/analysis/figures/dimension_energy-coverage-wakeupfor.pdf
diff --git a/analysis/figures/dimension_energy-coverage.pdf b/analysis/figures/dimension_energy-coverage.pdf
index 43e25f6..5acc33d 100644
--- a/analysis/figures/dimension_energy-coverage.pdf
+++ b/analysis/figures/dimension_energy-coverage.pdf
diff --git a/analysis/figures/dimension_energy.pdf b/analysis/figures/dimension_energy.pdf
index 8902663..ca4d59a 100644
--- a/analysis/figures/dimension_energy.pdf
+++ b/analysis/figures/dimension_energy.pdf
diff --git a/analysis/figures/months_knn.pdf b/analysis/figures/months_knn.pdf
new file mode 100644
index 0000000..f755f36
--- /dev/null
+++ b/analysis/figures/months_knn.pdf
diff --git a/analysis/figures/months_tree.pdf b/analysis/figures/months_tree.pdf
new file mode 100644
index 0000000..0e19ef4
--- /dev/null
+++ b/analysis/figures/months_tree.pdf
diff --git a/analysis/figures/sim_dimension_coverage_NO_HINT.pdf b/analysis/figures/sim_dimension_coverage_NO_HINT.pdf
index 2b5d154..0eae169 100644
--- a/analysis/figures/sim_dimension_coverage_NO_HINT.pdf
+++ b/analysis/figures/sim_dimension_coverage_NO_HINT.pdf
diff --git a/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf b/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf
index cc72a3e..73d64d7 100644
--- a/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf
+++ b/analysis/figures/sim_dimension_coverage_WITH_HINT.pdf
diff --git a/analysis/figures/sim_dimension_energy_NO_HINT.pdf b/analysis/figures/sim_dimension_energy_NO_HINT.pdf
index 048f1a8..142328a 100644
--- a/analysis/figures/sim_dimension_energy_NO_HINT.pdf
+++ b/analysis/figures/sim_dimension_energy_NO_HINT.pdf
diff --git a/analysis/figures/sim_dimension_energy_WITH_HINT.pdf b/analysis/figures/sim_dimension_energy_WITH_HINT.pdf
index d68df7b..e5898c5 100644
--- a/analysis/figures/sim_dimension_energy_WITH_HINT.pdf
+++ b/analysis/figures/sim_dimension_energy_WITH_HINT.pdf