Merge pull request #68 from edx/will/diff-cover-integration

Will/diff cover integration

doc/testing.md

@@ -202,9 +202,10 @@ To view test coverage:
 
 2. Generate reports:
 
-        rake coverage:html
+        rake coverage
 
-3. HTML reports are located in the `reports` folder.
+3. Reports are located in the `reports` folder.  The command
+generates HTML and XML (Cobertura format) reports.
 
 
 ## Testing using queue servers

jenkins/test.sh

@@ -84,7 +84,8 @@ rake phantomjs_jasmine_cms || TESTS_FAILED=1
 rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1
 rake phantomjs_jasmine_common/static/coffee || TESTS_FAILED=1
 
-rake coverage:xml coverage:html
+# Generate coverage reports
+rake coverage
 
 [ $TESTS_FAILED == '0' ]
 rake autodeploy_properties

rakefiles/tests.rake

@@ -45,6 +45,12 @@ end
 directory REPORT_DIR
 
 task :clean_test_files do
+
+    # Delete all files in the reports directory, while preserving
+    # the directory structure.
+    sh("find #{REPORT_DIR} -type f -print0 | xargs --no-run-if-empty -0 rm")
+
+    # Reset the test fixtures
     sh("git clean -fqdx test_root")
 end
 
@@ -81,12 +87,11 @@ TEST_TASK_DIRS = []
 end
 
 Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
-    task_name = "test_#{lib}"
 
     report_dir = report_dir_path(lib)
 
     desc "Run tests for common lib #{lib}"
-    task task_name => report_dir do
+    task "test_#{lib}"  => ["clean_test_files", report_dir] do
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
         cmd = "nosetests #{lib}"
         sh(run_under_coverage(cmd, lib)) do |ok, res|
@@ -95,10 +100,13 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
     end
     TEST_TASK_DIRS << lib
 
-    desc "Run tests for common lib #{lib} (without coverage)"
-    task "fasttest_#{lib}" do
-        sh("nosetests #{lib}")
-    end
+    # There used to be a fasttest_#{lib} command that ran without coverage.
+    # However, this is an inconsistent usage of "fast":
+    # When running tests for lms and cms, "fast" means skipping
+    # staticfiles collection, but still running under coverage.
+    # We keep the fasttest_#{lib} command for backwards compatibility,
+    # but make it an alias to the normal test command.
+    task "fasttest_#{lib}" => "test_#{lib}"
 end
 
 task :report_dirs
@@ -119,30 +127,35 @@ task :test do
     end
 end
 
-namespace :coverage do
-    desc "Build the html coverage reports"
-    task :html => :report_dirs do
-        TEST_TASK_DIRS.each do |dir|
-            report_dir = report_dir_path(dir)
+desc "Build the html, xml, and diff coverage reports"
+task :coverage => :report_dirs do
 
-            if !File.file?("#{report_dir}/.coverage")
-                next
-            end
+    found_coverage_info = false
 
-            sh("coverage html --rcfile=#{dir}/.coveragerc")
+    TEST_TASK_DIRS.each do |dir|
+        report_dir = report_dir_path(dir)
+
+        if !File.file?("#{report_dir}/.coverage")
+            next
+        else
+            found_coverage_info = true
         end
-    end
 
-    desc "Build the xml coverage reports"
-    task :xml => :report_dirs do
-        TEST_TASK_DIRS.each do |dir|
-            report_dir = report_dir_path(dir)
+        # Generate the coverage.py HTML report
+        sh("coverage html --rcfile=#{dir}/.coveragerc")
 
-            if !File.file?("#{report_dir}/.coverage")
-                next
-            end
-            # Why doesn't the rcfile control the xml output file properly??
-            sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc")
-        end
+        # Generate the coverage.py XML report
+        sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc")
+
+        # Generate the diff coverage HTML report, based on the XML report
+        sh("diff-cover #{report_dir}/coverage.xml --html-report #{report_dir}/diff_cover.html")
+
+        # Print the diff coverage report to the console
+        sh("diff-cover #{report_dir}/coverage.xml")
+        puts "\n"
+    end
+
+    if not found_coverage_info
+        puts "No coverage info found.  Run `rake test` before running `rake coverage`."
     end
 end

requirements/edx/github.txt

@@ -10,3 +10,4 @@
 # Our libraries:
 -e git+https://github.com/edx/XBlock.git@2144a25d#egg=XBlock
 -e git+https://github.com/edx/codejail.git@5fb5fa0#egg=codejail
+-e git+https://github.com/edx/diff-cover.git@v0.1.0#egg=diff_cover