1 import interfaces.Controller;
2 import interfaces.AbstractRobot;
3
4 import java.lang.*;
5 import java.util.*;
6
7
12 public class QLR3 extends Controller
13 {
14
17
18 private AbstractRobot robot;
20 private int STATES=4; private int ACTIONS=5; private int LEEWAY=6; private int REWARD_LEEWAY=1; private int SLEEP_TIME=500;
25 int STOP_THRESHOLD=1000;
26
27 private int[] sensors={Controller.SENSOR_TYPE_LIGHT,Controller.SENSOR_TYPE_LIGHT,Controller.SENSOR_TYPE_LIGHT};
29
30 private int S1=0;
32 private int S2=1;
33 private int S3=2;
34 private int S4=3;
35
36 private int A1=0;
38 private int A2=1;
39 private int A3=2;
40 private int A4=3;
41 private int A5=4;
42
43 private int state, newState, prevLight, newLight, reward, action;
45 private boolean running;
46
47 private double table[][]=new double[STATES][ACTIONS];
49
50
51
54
55 public void initController(AbstractRobot r)
56 {
57 robot=r;
58
59 initTable();
61 }
62
63 public int[] getSensors()
64 {
65 return sensors;
66 }
67
68 public void run()
69 {
70 running=true;
72
73 begin();
75 }
76
77 public void halt()
78 {
79 running=false;
82
83 robot.stopMoving();
85 }
86
87 public AbstractRobot getRobot()
88 {
89 return robot;
90 }
91
92
95
96
102 public int rand(int limit)
103 {
104 return (int) (Math.random()*(limit+1));
105 }
106
107
110 public void initTable()
111 {
112 for (int i=0;i<STATES;i++)
113 {
114 for (int j=0;j<ACTIONS;j++)
115 {
116 int x=rand(3);
117 table[i][j]=x;
118 }
119 }
120 }
121
122
125 public void updateTable()
126 {
127 table[state][action]=table[state][action]+reward;
129 }
130
131
136 public int Q(int STATE)
137 {
138 int highest=0;
139
140 for (int i=0;i<ACTIONS;i++)
141 {
142 if(table[STATE][i]>table[STATE][highest])
143 {
144 highest=i;
145 }
146 }
147
148 return highest;
149 }
150
151
154 public void begin()
155 {
156
158 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S1;}
159 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S2;}
160 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S3;}
161 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S4;}
162
163 newLight=robot.getSensor2();
165
166 while (running)
168 {
169 prevLight=newLight;
171
172 state=newState;
174
175 if(rand(9)==0){action=rand(ACTIONS-1);}
177
178 else{action=Q(state);}
180
181 if(action==A1){robot.forward();}
183 if(action==A2){robot.backward();}
184 if(action==A3){robot.right();}
185 if(action==A4){robot.left();}
186 if(action==A5){robot.stopMoving();}
187
188 try{sleep(SLEEP_TIME);}catch(Exception e){}
190
191 robot.stopMoving();
193
194 newLight=robot.getSensor2();
196
197 if (newLight > prevLight+REWARD_LEEWAY){reward=1;robot.beep();}
199 else if (newLight < prevLight){reward=-1;}
200 else {reward=0;}
201
202 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S1;}
204 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S2;}
205 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S3;}
206 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S4;}
207
208 updateTable();
210
211 if(robot.getSensor2()>STOP_THRESHOLD)
213 {
214 robot.beep();
216 robot.beep();
217 robot.beep();
218 break;
219 }
220 }
221 }
222 }
223