Added CodeResponse tests

common/lib/capa/capa/responsetypes.py

@@ -1202,7 +1202,7 @@ class CodeResponse(LoncapaResponse):
         '''
          Grader reply is a JSON-dump of the following dict
            { 'correct': True/False,
-             'score': # TODO -- Partial grading
+             'score': Numeric value (floating point is okay) to assign to answer
              'msg': grader_msg }
 
         Returns (valid_score_msg, correct, score, msg):

common/lib/xmodule/xmodule/tests/__init__.py

@@ -19,6 +19,7 @@ import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
+from datetime import datetime
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
 from xmodule.graders import Score, aggregate_scores
@@ -283,30 +284,61 @@ class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
     '''
+    @staticmethod
+    def make_queuestate(key, time):
+        timestr = datetime.strftime(time,'%Y%m%d%H%M%S')
+        return (key, timestr)
+
+    def test_is_queued(self):
+        ''' 
+        Simple test of whether LoncapaProblem knows when it's been queued
+        '''
+        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+        
+        answer_ids = sorted(test_lcp.get_question_answers().keys())
+        num_answers = len(answer_ids)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for i in range(num_answers):
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+        test_lcp.correct_map.update(cmap)
+
+        self.assertEquals(test_lcp.is_queued(), False)
+
+        # Now we queue the LCP
+        cmap = CorrectMap()
+        for i in range(num_answers):
+            queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+        test_lcp.correct_map.update(cmap)
+
+        self.assertEquals(test_lcp.is_queued(), True)
+
     def test_update_score(self):
+        '''
+        Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
+        '''
         problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
         test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the 'queued' state
-        old_cmap = CorrectMap()
         answer_ids = sorted(test_lcp.get_question_answers().keys())
-        numAnswers = len(answer_ids)
-        for i in range(numAnswers):
+        num_answers = len(answer_ids)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        old_cmap = CorrectMap()
+        for i in range(num_answers):
             queuekey = 1000 + i
-            queuestate = (queuekey, '')
+            queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
             old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
-        # TODO: Message format inherited from ExternalResponse
-        #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
-        #incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
-
-        # New message format common to external graders
+        # Message format common to external graders
         correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
         incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
 
         xserver_msgs = {'correct': correct_score_msg,
-                        'incorrect': incorrect_score_msg,
-                        }
+                        'incorrect': incorrect_score_msg,}
 
         # Incorrect queuekey, state should not be updated
         for correctness in ['correct', 'incorrect']:
@@ -316,12 +348,12 @@ class CodeResponseTest(unittest.TestCase):
             test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
             self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-            for i in range(numAnswers):
+            for i in range(num_answers):
                 self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
 
         # Correct queuekey, state should be updated
         for correctness in ['correct', 'incorrect']:
-            for i in range(numAnswers):  # Target specific answer_id's
+            for i in range(num_answers):  # Target specific answer_id's
                 test_lcp.correct_map = CorrectMap()
                 test_lcp.correct_map.update(old_cmap)
 
@@ -333,13 +365,51 @@ class CodeResponseTest(unittest.TestCase):
                 test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                 self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
 
-                for j in range(numAnswers):
+                for j in range(num_answers):
                     if j == i:
                         self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
                     else:
                         self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
+    
+
+    def test_recentmost_queuetime(self):
+        '''
+        Test whether the LoncapaProblem knows about the time of queue requests
+        '''
+        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+
+        answer_ids = sorted(test_lcp.get_question_answers().keys())
+        num_answers = len(answer_ids)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for i in range(num_answers):
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+        test_lcp.correct_map.update(cmap)
+        
+        self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        cmap = CorrectMap()
+        answer_ids = sorted(test_lcp.get_question_answers().keys())
+        num_answers = len(answer_ids)
+        for i in range(num_answers):
+            queuekey = 1000 + i
+            latest_timestamp = datetime.now()
+            queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+        test_lcp.correct_map.update(cmap)
+
+        # Queue state only tracks up to second
+        latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp,'%Y%m%d%H%M%S'),'%Y%m%d%H%M%S')
+
+        self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
 
     def test_convert_files_to_filenames(self):
+        '''
+        Test whether file objects are converted to filenames without altering other structures
+        '''
         problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
         fp = open(problem_file)
         answers_with_file = {'1_2_1': 'String-based answer',

common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml

@@ -9,91 +9,23 @@
 Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 
 <text>
-Write a program to compute the cube of a number
+Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 

common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml

+<problem>
+<text>
+<h2>Code response</h2>
+
+<p>
+</p>
+
+<text>
+Write a program to compute the square of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def square(n):
+"""
+
+answer = """
+def square(n):
+    return n**2    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testSquare(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: square(%d)'%n
+    return str(square(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testSquare(0))
+   elif test == 2: f.write(testSquare(1))
+   else:  f.write(testSquare())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+<text>
+Write a program to compute the cube of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def cube(n):
+"""
+
+answer = """
+def cube(n):
+    return n**3    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testCube(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: cube(%d)'%n
+    return str(cube(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testCube(0))
+   elif test == 2: f.write(testCube(1))
+   else:  f.write(testCube())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+</text>
+</problem>

lms/envs/test.py

@@ -58,7 +58,7 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
-
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # TODO (cpennington): We need to figure out how envs/test.py can inject things
 # into common.py so that we don't have to repeat this sort of thing