1 import interfaces.Controller;
2 import interfaces.AbstractRobot;
3
4 import java.lang.*;
5 import java.util.*;
6
7
13 public class InverseQLR3 extends Controller
14 {
15
18
19 private AbstractRobot robot;
21 private int STATES=4; private int ACTIONS=5; private int LEEWAY=6; private int REWARD_LEEWAY=1; private int SLEEP_TIME=500;
26 int STOP_THRESHOLD=1000;
27
28 private int[] sensors={Controller.SENSOR_TYPE_LIGHT,Controller.SENSOR_TYPE_LIGHT,Controller.SENSOR_TYPE_LIGHT};
30
31 private int S1=0;
33 private int S2=1;
34 private int S3=2;
35 private int S4=3;
36
37 private int A1=0;
39 private int A2=1;
40 private int A3=2;
41 private int A4=3;
42 private int A5=4;
43
44 private int state, newState, prevLight, newLight, reward, action;
46 private boolean running;
47
48 private double table[][]=new double[STATES][ACTIONS];
50
51
52
55
56 public void initController(AbstractRobot r)
57 {
58 robot=r;
59
60 initTable();
62 }
63
64 public int[] getSensors()
65 {
66 return sensors;
67 }
68
69 public void run()
70 {
71 running=true;
73
74 begin();
76 }
77
78 public void halt()
79 {
80 running=false;
83
84 robot.stopMoving();
86 }
87
88 public AbstractRobot getRobot()
89 {
90 return robot;
91 }
92
93
96
97
103 public int rand(int limit)
104 {
105 return (int) (Math.random()*(limit+1));
106 }
107
108
111 public void initTable()
112 {
113 for (int i=0;i<STATES;i++)
114 {
115 for (int j=0;j<ACTIONS;j++)
116 {
117 int x=rand(3);
118 table[i][j]=x;
119 }
120 }
121 }
122
123
126 public void updateTable()
127 {
128 table[state][action]=table[state][action]+reward;
130 }
131
132
137 public int Q(int STATE)
138 {
139 int highest=0;
140
141 for (int i=0;i<ACTIONS;i++)
142 {
143 if(table[STATE][i]>table[STATE][highest])
144 {
145 highest=i;
146 }
147 }
148
149 return highest;
150 }
151
152
155 public void begin()
156 {
157 int exploreRate=9;
158
159
161 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S1;}
162 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S2;}
163 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S3;}
164 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S4;}
165
166 newLight=robot.getSensor2();
168
169
173 while (running)
175 {
176
177 prevLight=newLight;
179
180 state=newState;
182
183 if(rand(exploreRate)==0)
186 {
187 action=rand(ACTIONS-1);
188 exploreRate++;
190 }
191 else{action=Q(state);}
193
194 if(action==A1){robot.forward();}
196 if(action==A2){robot.backward();}
197 if(action==A3){robot.right();}
198 if(action==A4){robot.left();}
199 if(action==A5){robot.stopMoving();}
200
201 try{sleep(SLEEP_TIME);}catch(Exception e){}
203
204 robot.stopMoving();
206
207 newLight=robot.getSensor2();
209
210 if (newLight > prevLight+REWARD_LEEWAY){reward=-1;robot.beep();}
212 else if (newLight < prevLight){reward=1;}
213 else {reward=0;}
214
215 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S1;}
217 if(robot.getSensor1() > robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S2;}
218 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() > robot.getSensor3()){newState=S3;}
219 if(robot.getSensor1() < robot.getSensor2() && robot.getSensor2() < robot.getSensor3()){newState=S4;}
220
221 updateTable();
223
224 if(robot.getSensor2()>STOP_THRESHOLD)
226 {
227 robot.beep();
229 robot.beep();
230 robot.beep();
231 break;
232 }
233 }
234 }
235 }
236