1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
library("tidyverse")
library("class")
library("rpart")
library("rpart.plot")
library("viridis")
library("MLmetrics")
## Simulation Parameters:
## simkey {baseline,extended,hint,hintandextended}
## wireless {lora,nbiot}
## wakeupfor {60s,180s}
## seed [1,200]
## node on[0,12]
## isSender {0,1}
## dataSize {1MB}
## Metrics:
## energy [0,+inf)
## nDataRcv [0,+inf)
nseed=200
nwakeupfor=2
nwireless=2
nsimkey=4
nsimulations=nseed*nwakeupfor*nwireless*nsimkey # Must be 3200
## Load data
data=read_csv("../CCGRID2022.csv")%>%distinct() # Note that in the data experiment wireless=="lora",seed==1,wakeupfor==60,simkey=="baseline" is present 2 times in the CSV file
tmp_data_coverage=data%>%group_by(simkey,wireless,wakeupfor,seed)%>%mutate(coverage=sum(nDataRcv))%>%ungroup()%>%filter(isSender==1)%>%select(simkey,wireless,wakeupfor,seed,coverage)
data_seed_isSender=data%>%group_by(simkey,wireless,wakeupfor,seed,isSender)%>%summarize(energy_mean=mean(energy))%>%
left_join(tmp_data_coverage,by=c("simkey","wireless","wakeupfor","seed"))%>%
mutate(efficiency=energy_mean/coverage)%>%
ungroup()
data_seed=data%>%group_by(simkey,wireless,wakeupfor,seed)%>%summarize(energy=sum(energy),coverage=sum(nDataRcv))%>%
mutate(efficiency=energy/coverage)%>%
ungroup()
generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl="lora",wuf=180) {
attempts=seq(1,attempts_max)
results=sapply(attempts,function(attempt){
## Prepare data for traning
set.seed(1+attempt) # Reproducibility
wireless_map=c("lora"=1,"nbiot"=2)
cur_data_seed=data_seed%>%filter(wakeupfor==wuf,wireless==wrl)
data_ml=cur_data_seed%>%select(-efficiency)%>%mutate(wireless=wireless_map[cur_data_seed$wireless])
if(ignore_hint){
data_ml=data_ml%>%filter(simkey!="hint")
}
train_set=data_ml%>%filter(seed<=seed_max)%>%select(-seed) # train data on seed_max*3 days
test_set=data_ml%>%anti_join(train_set)%>%select(-seed) # build test_sed excluding training set
## KNN training
knn_predictions=knn(train=train_set%>%select(-simkey),test=test_set%>%select(-simkey),cl=train_set$simkey,k=min(10,NROW(train_set)))
## KNN analysis
knn_cont_table=table(knn_predictions,test_set$simkey)
knn_accuracy=round((sum(diag(knn_cont_table)/sum(rowSums(knn_cont_table))))*100)
knn_prop_table=round(prop.table(knn_cont_table),digits=2)
knn_f1_score=F1_Score(test_set$simkey,knn_predictions)
knn_recall=Recall(test_set$simkey,knn_predictions)
knn_precision=Precision(test_set$simkey,knn_predictions)
## Decision tree
tree=rpart(
simkey ~ wireless + wakeupfor + energy + coverage,
data=train_set,
method="class",
minsplit=60,
minbucket=1)
tree_predictions=predict(tree,newdata=test_set%>%select(-simkey),type="class")
tree_cont_table=table(tree_predictions,test_set$simkey)
tree_accuracy=round((sum(diag(tree_cont_table)/sum(rowSums(tree_cont_table))))*100)
tree_prop_table=round(prop.table(tree_cont_table),digits=2)
tree_f1_score=F1_Score(test_set$simkey,tree_predictions)
tree_recall=Recall(test_set$simkey,tree_predictions)
tree_precision=Precision(test_set$simkey,tree_predictions)
list(tibble(seed_max=seed_max,model=c("knn","tree"),accuracy=c(knn_accuracy,tree_accuracy),f1_score=c(knn_f1_score,tree_f1_score),recall=c(knn_recall,tree_recall),precision=c(knn_precision,tree_precision)))
})
## Prints
results=do.call("rbind",results)
results%>%mutate(seed_max=seed_max,attempts_max=attempts_max,wireless=wrl,wakeupfor=wuf)
}
generate_accuracy = function(wireless,wakeupfor,steps=1, accuracy=20,ignore_hint=TRUE){
npolicies=4
if(ignore_hint){npolicies=npolicies-1}
## Generate inputs
result=tibble()
for(i in seq(1,160,by=steps)){ # We stop at 80% of the data (this way test set is at least 20%)
acc=generate_accuracy_for(ignore_hint=ignore_hint,seed=i,attempts_max=accuracy,wrl=wireless,wuf=wakeupfor)
result=rbind(result,acc)
}
result%>%mutate(days=seed_max*npolicies,months=days/30) # Since 3 policies (since ignore_hint=TRUE)
}
# Generate accuracy for each wireless and uptime
accuracy=rbind(generate_accuracy("lora",60),
generate_accuracy("lora",180),
generate_accuracy("nbiot",60),
generate_accuracy("nbiot",180))
## Summarize
result_summary=accuracy%>%group_by(wireless,wakeupfor,months,model)%>%
summarize(
mean_accuracy=mean(accuracy),sd_accuracy=sd(accuracy),min_accuracy=min(accuracy),max_accuracy=max(accuracy),
mean_f1_score=mean(f1_score),sd_f1_score=sd(f1_score),min_f1_score=min(f1_score),max_f1_score=max(f1_score),
mean_recall=mean(recall),sd_recall=sd(recall),min_recall=min(recall),max_recall=max(recall),
mean_precision=mean(precision),sd_precision=sd(precision),min_precision=min(precision),max_precision=max(precision))
## Result max
metrics_peak=result_summary%>%group_by(wireless,wakeupfor,model)%>%
summarize(max_accuracy=max(mean_accuracy),
max_f1_score=max(mean_f1_score),
max_recall=max(mean_recall),
max_precision=max(mean_precision))
## Plot
sapply(c("knn","tree"),function(grp){
ggplot(result_summary%>%filter(model==grp),aes(months,mean_accuracy))+
geom_ribbon(aes(ymin=mean_accuracy-sd_accuracy,ymax=mean_accuracy+sd_accuracy),alpha=0.2,color=NA)+
geom_line(size=1.1)+geom_point(size=3)+xlab("Number of training months")+ylab(paste("Mean",grp,"accuracy"))+ggtitle(paste(grp,"accuracy"))+
# ylim(c(0,100))+
geom_hline(data=metrics_peak%>%filter(model==grp),aes(yintercept=max_accuracy),color="red",size=1)+
geom_text(data=metrics_peak%>%filter(model==grp),x=0,aes(y=max_accuracy,label = round(max_accuracy,digits=1),vjust=-1),color="red")+
facet_wrap(~wireless+wakeupfor)
scale_x_continuous(breaks = seq(0, max(result_summary$months), by = 1))
ggsave(paste0("figures/months_",grp,".pdf"))
})
|
