Add BIST tests for octave functions written in C++

From Octave
Revision as of 10:46, 12 December 2015 by Mtmiller (talk | contribs) (→‎Instances: done with getrusage)
Jump to navigation Jump to search

Introduction

As new features are added to Octave we need to ensure that we don't break existing functionality. At the heart of that promise not to break functionality lies regression testing. Regression testing is done practically daily and certainly whenever a signicant change is committed. The core of the regression tests are Built-In Self Tests (BIST) which are tests written in the Octave language itself that are placed in comments just below the code being tested. The full documentation for these tests is in Appendix B of the manual (Test Functions). While BIST tests exist for most m-file functions, many core functions written in C++ lack these BIST tests. The goal of this sprint topic is to significantly reduce the number of uncovered core functions.

The syntax for writing tests is documented in Appendix B. The general form of the tests is

  1. Basic functionality
    • Does it work for all different combinations of number of inputs and number of outputs
    • Does it work for all expected input classes (matrices, strings, logicals, cell arrays, structs?)
  2. Corner Case functionality
    • Does it work for corner cases (empty matrices, Inf or NaN, etc.)
    • Tests for specific bugs that have been reported on the bug tracker
  3. Input validation
    • Does it correctly generate errors or warnings for bad inputs

Example 1 : Simple Function (rows() from data.cc)

C++ Code

  octave_value retval;

  if (args.length () == 1)
    retval = args(0).rows ();
  else
    print_usage ();

  return retval;

BIST tests

/*
%!assert (rows (ones (2,5)), 2)
%!assert (rows (ones (5,2)), 5)
%!assert (rows (ones (5,4,3,2)), 2)
%!assert (rows (ones (3,4,5,2)), 3)

%!assert (rows (cell (2,5)), 2)
%!assert (rows (cell (5,2)), 5)
%!assert (rows (cell (5,4,3,2)), 2)
%!assert (rows (cell (3,4,5,2)), 3)

%!test
%! x(2,5,3).a = 1;
%! assert (rows (x), 2);
%! y(5,4,3).b = 2;
%! assert (rows (y), 5);

%!assert (rows ("Hello World"), 1)

%!assert (rows ([]), 0)
%!assert (rows (zeros (2,0), 2)

## Test input validation
%!error rows ()
%!error rows (1,2)
*/

Example 2 : Complicated input validation (issorted() from data.cc)

C++ Code

  octave_value retval;

  int nargin = args.length ();

  if (nargin < 1 || nargin > 3)
    {
      print_usage ();
      return retval;
    }

  bool by_rows = false;

  sortmode smode = ASCENDING;

  if (nargin > 1)
    {
      octave_value mode_arg;

      if (nargin == 3)
        smode = get_sort_mode_option (args(2));

      std::string tmp = args(1).xstring_value ("issorted: second argument must be a string");
      if (tmp == "rows")
        by_rows = true;
      else
        smode = get_sort_mode_option (args(1));
    }

  octave_value arg = args(0);

  if (by_rows)
    {
      if (arg.is_sparse_type ())
        error ("issorted: sparse matrices not yet supported");
      if (arg.ndims () == 2)
        retval = arg.is_sorted_rows (smode) != UNSORTED;
      else
        error ("issorted: A must be a 2-dimensional object");
    }
  else
    {
      if (arg.dims ().is_vector ())
        retval = args(0).is_sorted (smode) != UNSORTED;
      else
        error ("issorted: needs a vector");
    }

  return retval;
}

BIST tests

/*
%!shared sm, um, sv, uv
%! sm = [1, 2; 3, 4];
%! um = [3, 1; 2, 4];
%! sv = [1, 2, 3, 4];
%! uv = [2, 1, 4, 3];

%!assert (issorted (sm, "rows"))
%!assert (! issorted (um, "rows"))
%!assert (issorted (sv))
%!assert (! issorted (uv))
%!assert (issorted (sv'))
%!assert (! issorted (uv'))
%!assert (issorted (sm, "rows", "ascending"))
%!assert (! issorted (um, "rows", "ascending"))
%!assert (issorted (sv, "ascending"))
%!assert (! issorted (uv, "ascending"))
%!assert (issorted (sv', "ascending"))
%!assert (! issorted (uv', "ascending"))
%!assert (! issorted (sm, "rows", "descending"))
%!assert (issorted (flipud (sm), "rows", "descending"))
%!assert (! issorted (sv, "descending"))
%!assert (issorted (fliplr (sv), "descending"))
%!assert (! issorted (sv', "descending"))
%!assert (issorted (fliplr (sv)', "descending"))
%!assert (! issorted (um, "rows", "either"))
%!assert (! issorted (uv, "either"))
%!assert (issorted (sm, "rows", "either"))
%!assert (issorted (flipud (sm), "rows", "either"))
%!assert (issorted (sv, "either"))
%!assert (issorted (fliplr (sv), "either"))
%!assert (issorted (sv', "either"))
%!assert (issorted (fliplr (sv)', "either"))

%!error <needs a vector> issorted ([])

## Test input validation
%!error issorted () 
%!error issorted (1,2,3,4) 
%!error <second argument must be a string> issorted (1, 2)
%!error <second argument must be a string> issorted (1, {"rows"})
%!error <sparse matrices not yet supported> issorted (sparse ([1 2 3]), "rows")
%!error <A must be a 2-dimensional object> issorted (rand (2,2,2), "rows")
%!error <needs a vector> issorted (ones (2,2))
*/

Example 3 : Interactive or otherwise untestable function (keyboard() from input.cc)

C++ Code

  octave_value_list retval;

  int nargin = args.length ();

  if (nargin == 0 || nargin == 1)
    {
      unwind_protect frame;

      frame.add_fcn (octave_call_stack::restore_frame,
                     octave_call_stack::current_frame ());

      // Skip the frame assigned to the keyboard function.
      octave_call_stack::goto_frame_relative (0);

      tree_evaluator::debug_mode = true;
      tree_evaluator::quiet_breakpoint_flag = false;

      tree_evaluator::current_frame = octave_call_stack::current_frame ();

      do_keyboard (args);
    }
  else
    print_usage ();

  return retval;

BIST tests

For interactive functions there may be no way to actually test the function. In this case it is appropriate to mark the function so it will no longer show up as untested in later reports. Use the following code to do that.

/*
%!# No BIST tests possible for keyboard function.
*/

But, before you absolutely disable testing for a function, look to see if you can at least do input validation tests. In this case,

/*
%!error keyboard (1,2)
*/

Detailed Instructions

The list of undocumented C++ functions is in the Instances section. Tests should be enclosed by '/*' and '*/' multi-line comment markers. The comment block should have one newline above and below it to separate it from surrounding code.

To avoid duplication, sign up for a particular function by editing the Instances section of this wiki page and replacing '???' with your name. When you have edited a file you should verify that everything is okay by executing

make all
./run-octave
test FILENAME.cc

When that passes, let a Maintainer know so that we can check in the changes. Also, add the wiki tags

<strike> ... </strike>

to the Instances section to cross the file off the list. In addition, increment the number of documented functions by +1.

Instances

Start of Sprint

Total: 280

Fixed: 11

Owner File
??? corefcn/balance.cc:balance
mtmx corefcn/besselj.cc:besselh (marked as already tested)
mtmx corefcn/besselj.cc:besseli (marked as already tested)
mtmx corefcn/besselj.cc:besselj (marked as already tested)
mtmx corefcn/besselj.cc:besselk (marked as already tested)
mtmx corefcn/besselj.cc:bessely (marked as already tested)
mtmx corefcn/bitfcns.cc:bitand (bitand, bitor, and bitxor are tested together)
mtmx corefcn/bitfcns.cc:bitor (bitand, bitor, and bitxor are tested together)
mtmx corefcn/bitfcns.cc:flintmax
mtmx corefcn/bitfcns.cc:intmax
mtmx corefcn/bitfcns.cc:intmin
mtmx corefcn/bitfcns.cc:sizemax
??? corefcn/cellfun.cc:cellindexmat
??? corefcn/colloc.cc:colloc
??? corefcn/daspk.cc:daspk
??? corefcn/dasrt.cc:dasrt
??? corefcn/data.cc:I
??? corefcn/data.cc:and
??? corefcn/data.cc:colon
??? corefcn/data.cc:columns
??? corefcn/data.cc:complex
??? corefcn/data.cc:cputime
??? corefcn/data.cc:e
??? corefcn/data.cc:eq
??? corefcn/data.cc:false
??? corefcn/data.cc:full
??? corefcn/data.cc:ge
??? corefcn/data.cc:gt
??? corefcn/data.cc:ipermute
??? corefcn/data.cc:iscomplex
??? corefcn/data.cc:isfloat
??? corefcn/data.cc:isinteger
??? corefcn/data.cc:isreal
??? corefcn/data.cc:ldivide
??? corefcn/data.cc:le
??? corefcn/data.cc:length
??? corefcn/data.cc:lt
??? corefcn/data.cc:merge
??? corefcn/data.cc:minus
??? corefcn/data.cc:mldivide
??? corefcn/data.cc:mpower
??? corefcn/data.cc:mrdivide
??? corefcn/data.cc:mtimes
??? corefcn/data.cc:ndims
??? corefcn/data.cc:ne
??? corefcn/data.cc:nnz
??? corefcn/data.cc:not
??? corefcn/data.cc:nth_element
??? corefcn/data.cc:numel
??? corefcn/data.cc:nzmax
??? corefcn/data.cc:or
??? corefcn/data.cc:permute
??? corefcn/data.cc:pi
??? corefcn/data.cc:plus
??? corefcn/data.cc:power
??? corefcn/data.cc:rdivide
??? corefcn/data.cc:realmax
??? corefcn/data.cc:realmin
??? corefcn/data.cc:repelems
??? corefcn/data.cc:resize
??? corefcn/data.cc:rows
??? corefcn/data.cc:size
??? corefcn/data.cc:size_equal
??? corefcn/data.cc:squeeze
??? corefcn/data.cc:tic
??? corefcn/data.cc:times
??? corefcn/data.cc:true
??? corefcn/data.cc:uminus
??? corefcn/data.cc:uplus
??? corefcn/debug.cc:dbclear
??? corefcn/debug.cc:dbcont
??? corefcn/debug.cc:dbdown
??? corefcn/debug.cc:dblist
??? corefcn/debug.cc:dbquit
??? corefcn/debug.cc:dbstack
??? corefcn/debug.cc:dbstatus
??? corefcn/debug.cc:dbstep
??? corefcn/debug.cc:dbstop
??? corefcn/debug.cc:dbtype
??? corefcn/debug.cc:dbup
??? corefcn/debug.cc:dbwhere
??? corefcn/debug.cc:isdebugmode
??? corefcn/dirfns.cc:cd
??? corefcn/dirfns.cc:confirm_recursive_rmdir
??? corefcn/dirfns.cc:filesep
??? corefcn/dirfns.cc:pathsep
??? corefcn/dirfns.cc:pwd
??? corefcn/dirfns.cc:readdir
??? corefcn/error.cc:beep_on_error
??? corefcn/error.cc:debug_on_error
??? corefcn/error.cc:debug_on_warning
??? corefcn/error.cc:error
??? corefcn/error.cc:lasterr
??? corefcn/error.cc:lasterror
??? corefcn/error.cc:lastwarn
??? corefcn/error.cc:rethrow
??? corefcn/error.cc:warning
??? corefcn/fft.cc:fft
??? corefcn/fft2.cc:fft2
??? corefcn/fftn.cc:fftn
??? corefcn/fftn.cc:ifftn
??? corefcn/file-io.cc:fclear
??? corefcn/file-io.cc:fclose
??? corefcn/file-io.cc:fflush
??? corefcn/file-io.cc:fgetl
??? corefcn/file-io.cc:fgets
??? corefcn/file-io.cc:fopen
??? corefcn/file-io.cc:fprintf
??? corefcn/file-io.cc:fputs
??? corefcn/file-io.cc:fread
??? corefcn/file-io.cc:freport
??? corefcn/file-io.cc:frewind
??? corefcn/file-io.cc:fscanf
??? corefcn/file-io.cc:fseek
??? corefcn/file-io.cc:fskipl
??? corefcn/file-io.cc:ftell
??? corefcn/file-io.cc:fwrite
??? corefcn/file-io.cc:mkstemp
??? corefcn/file-io.cc:printf
??? corefcn/file-io.cc:puts
??? corefcn/file-io.cc:scanf
??? corefcn/file-io.cc:sprintf
??? corefcn/file-io.cc:sscanf
??? corefcn/file-io.cc:tmpfile
??? corefcn/getgrent.cc:endgrent
??? corefcn/getgrent.cc:getgrent
??? corefcn/getgrent.cc:getgrgid
??? corefcn/getgrent.cc:getgrnam
??? corefcn/getgrent.cc:setgrent
??? corefcn/getpwent.cc:endpwent
??? corefcn/getpwent.cc:getpwent
??? corefcn/getpwent.cc:getpwnam
??? corefcn/getpwent.cc:getpwuid
??? corefcn/getpwent.cc:setpwent
mtmx corefcn/getrusage.cc:getrusage
??? corefcn/graphics.cc:addlistener
??? corefcn/graphics.cc:addproperty
??? corefcn/graphics.cc:available_graphics_toolkits
??? corefcn/graphics.cc:dellistener
??? corefcn/graphics.cc:drawnow
??? corefcn/graphics.cc:ishandle
??? corefcn/graphics.cc:loaded_graphics_toolkits
??? corefcn/graphics.cc:register_graphics_toolkit
??? corefcn/graphics.cc:set
??? corefcn/graphics.cc:waitfor
??? corefcn/help.cc:built_in_docstrings_file
??? corefcn/help.cc:doc_cache_file
??? corefcn/help.cc:get_help_text
??? corefcn/help.cc:get_help_text_from_file
??? corefcn/help.cc:info_file
??? corefcn/help.cc:info_program
??? corefcn/help.cc:makeinfo_program
??? corefcn/help.cc:suppress_verbose_help_message
??? corefcn/help.cc:texi_macros_file
??? corefcn/input.cc:PS1
??? corefcn/input.cc:PS2
??? corefcn/input.cc:PS4
??? corefcn/input.cc:add_input_event_hook
??? corefcn/input.cc:completion_append_char
??? corefcn/input.cc:echo_executing_commands
??? corefcn/input.cc:filemarker
??? corefcn/input.cc:input
??? corefcn/input.cc:keyboard
??? corefcn/input.cc:readline_re_read_init_file
??? corefcn/input.cc:readline_read_init_file
??? corefcn/input.cc:remove_input_event_hook
??? corefcn/input.cc:yes_or_no
??? corefcn/inv.cc:inverse
??? corefcn/load-path.cc:addpath
??? corefcn/load-path.cc:command_line_path
??? corefcn/load-path.cc:genpath
??? corefcn/load-path.cc:path
??? corefcn/load-path.cc:rehash
??? corefcn/load-path.cc:restoredefaultpath
??? corefcn/load-path.cc:rmpath
??? corefcn/load-save.cc:crash_dumps_octave_core
??? corefcn/load-save.cc:load
??? corefcn/load-save.cc:octave_core_file_limit
??? corefcn/load-save.cc:octave_core_file_name
??? corefcn/load-save.cc:octave_core_file_options
??? corefcn/load-save.cc:save
??? corefcn/load-save.cc:save_default_options
??? corefcn/load-save.cc:save_header_format_string
??? corefcn/ls-oct-text.cc:save_precision
??? corefcn/mappers.cc:angle
??? corefcn/oct-hist.cc:edit_history
??? corefcn/oct-hist.cc:history
??? corefcn/oct-hist.cc:history_control
??? corefcn/oct-hist.cc:history_file
??? corefcn/oct-hist.cc:history_save
??? corefcn/oct-hist.cc:history_size
??? corefcn/oct-hist.cc:history_timestamp_format_string
??? corefcn/oct-hist.cc:run_history
??? corefcn/pager.cc:PAGER
??? corefcn/pager.cc:PAGER_FLAGS
??? corefcn/pager.cc:diary
??? corefcn/pager.cc:more
??? corefcn/pager.cc:page_output_immediately
??? corefcn/pager.cc:page_screen_output
??? corefcn/pager.cc:terminal_size
??? corefcn/pr-output.cc:disp
??? corefcn/pr-output.cc:fixed_point_format
??? corefcn/pr-output.cc:format
??? corefcn/pr-output.cc:output_max_field_width
??? corefcn/pr-output.cc:output_precision
??? corefcn/pr-output.cc:print_empty_dimensions
??? corefcn/pr-output.cc:rats
??? corefcn/pr-output.cc:split_long_rows
??? corefcn/pt-jit.cc:debug_jit
??? corefcn/pt-jit.cc:jit_enable
??? corefcn/pt-jit.cc:jit_failcnt
??? corefcn/pt-jit.cc:jit_startcnt
??? corefcn/sparse.cc:issparse
??? corefcn/sparse.cc:spalloc
??? corefcn/sparse.cc:sparse
??? corefcn/sub2ind.cc:ind2sub
??? corefcn/symtab.cc:set_variable
??? corefcn/syscalls.cc:gethostname
??? corefcn/syscalls.cc:uname
??? corefcn/sysdep.cc:clc
??? corefcn/sysdep.cc:have_window_system
??? corefcn/sysdep.cc:kbhit
??? corefcn/sysdep.cc:unsetenv
??? corefcn/toplev.cc:atexit
??? corefcn/toplev.cc:quit
??? corefcn/toplev.cc:warranty
??? corefcn/tril.cc:tril
??? corefcn/typecast.cc:bitpack
??? corefcn/typecast.cc:bitunpack
??? corefcn/typecast.cc:typecast
??? corefcn/urlwrite.cc:urlread
??? corefcn/urlwrite.cc:urlwrite
??? corefcn/variables.cc:clear
??? corefcn/variables.cc:isglobal
??? corefcn/variables.cc:mislocked
??? corefcn/variables.cc:missing_component_hook
??? corefcn/variables.cc:missing_function_hook
??? corefcn/variables.cc:mlock
??? corefcn/variables.cc:munlock
??? corefcn/variables.cc:who
??? corefcn/variables.cc:whos
??? corefcn/variables.cc:whos_line_format
??? octave-value/ov-cell.cc:cell
??? octave-value/ov-cell.cc:cellstr
??? octave-value/ov-cell.cc:iscell
??? octave-value/ov-cell.cc:iscellstr
??? octave-value/ov-class.cc:inferiorto
??? octave-value/ov-class.cc:ismethod
??? octave-value/ov-class.cc:isobject
??? octave-value/ov-class.cc:superiorto
??? octave-value/ov-classdef.cc.wip:metaclass
??? octave-value/ov-classdef.cc:metaclass
??? octave-value/ov-fcn-handle.cc:func2str
??? octave-value/ov-fcn-handle.cc:functions
??? octave-value/ov-java.cc:debug_java
??? octave-value/ov-java.cc:java2mat
??? octave-value/ov-java.cc:java_matrix_autoconversion
??? octave-value/ov-java.cc:java_unsigned_autoconversion
??? octave-value/ov-struct.cc:isfield
??? octave-value/ov-struct.cc:isstruct
??? octave-value/ov-struct.cc:print_struct_array_contents
??? octave-value/ov-struct.cc:struct_levels_to_print
??? octave-value/ov-usr-fcn.cc:nargin
??? octave-value/ov-usr-fcn.cc:nargout
??? octave-value/ov-usr-fcn.cc:optimize_subsasgn_calls
??? octave-value/ov.cc:subsref
??? parse-tree/oct-parse.cc:assignin
??? parse-tree/oct-parse.cc:autoload
??? parse-tree/oct-parse.cc:builtin
??? parse-tree/oct-parse.cc:evalin
??? parse-tree/oct-parse.cc:feval
??? parse-tree/oct-parse.cc:mfilename
??? parse-tree/oct-parse.cc:source
??? parse-tree/oct-parse.in.yy:assignin
??? parse-tree/oct-parse.in.yy:autoload
??? parse-tree/oct-parse.in.yy:builtin
??? parse-tree/oct-parse.in.yy:evalin
??? parse-tree/oct-parse.in.yy:feval
??? parse-tree/oct-parse.in.yy:mfilename
??? parse-tree/oct-parse.in.yy:source