<?xml version="1.0" encoding="UTF-8"?>
<!--
 * Scilab ( https://www.scilab.org/ ) - This file is part of Scilab
 * Copyright (C) INRIA
 *
 * Copyright (C) 2012 - 2016 - Scilab Enterprises
 *
 * This file is hereby licensed under the terms of the GNU GPL v2.0,
 * pursuant to article 5.3.4 of the CeCILL v.2.1.
 * This file was originally licensed under the terms of the CeCILL v2.1,
 * and continues to be available under such terms.
 * For more information, see the COPYING file which you should have received
 * along with this program.
 *
 -->
<refentry xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svg="http://www.w3.org/2000/svg" xmlns:ns5="http://www.w3.org/1999/xhtml" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:db="http://docbook.org/ns/docbook" xmlns:scilab="http://www.scilab.org" xml:id="bench_run" xml:lang="en">
    <refnamediv>
        <refname>bench_run</refname>
        <refpurpose>Launches benchmark tests</refpurpose>
    </refnamediv>
    <refsynopsisdiv>
        <title>Syntax</title>
        <synopsis>
            [modutests_names, elapsed_time, nb_iterations] = bench_run()
            [modutests_names, elapsed_time, nb_iterations] = bench_run(module[, test_name[, options, [exportToFile]])
            [modutests_names, elapsed_time, nb_iterations] = bench_run(path_to_module[, test_name[, options, [exportToFile]])
        </synopsis>
    </refsynopsisdiv>
    <refsection>
        <title>Arguments</title>
        <variablelist>
            <varlistentry>
                <term>module</term>
                <listitem>
                    <para>a vector of string. Contains the names of a Scilab modules to benchmark.</para>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>path_to_module</term>
                <listitem>
                    <para>
                        a vector of string. Contains the paths to directories of modules to test. If <literal>"/path/to/directory"</literal> is given as input parameter, tests are retrieved in the subdirectory
                        <literal>
                            /path/to/directory/<emphasis role="bold">tests/benchmarks</emphasis>
                        </literal>
                        .Used for homemade benchmarks.
                    </para>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>test_name</term>
                <listitem>
                    <para>a vector of string. Contains the names of the tests to perform.</para>
                    <para>
                        The name of a test is its filename without <literal>.tst</literal>.
                        If several modules or directory are given as first input parameter,
                        scans for tests in each of these modules or directory.
                    </para>
                    <note>
                        Partial test names are allowed to run a subset
                        of benchmarks dedicated to the same function/feature.
                        For instance, specifying <literal>"ascii"</literal>
                        will select all tests (in given module(s)) whose names
                        contain <literal>"ascii"</literal> (See examples).
                    </note>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>options</term>
                <listitem>
                    <para>a vector of string</para>
                    <itemizedlist>
                        <listitem>
                            <para>
                                <literal>"list"</literal>: list of the benchmark tests (<literal>test_name</literal>) available in a module
                            </para>
                        </listitem>
                        <listitem>
                            <para>
                                <literal>"help"</literal>: displays some examples of use in the Scilab console
                            </para>
                        </listitem>
                        <listitem>
                            <para>
                                <literal>"nb_run=value"</literal>: runs each benchmark <literal>value</literal> times ; by default <function>bench_run</function> runs 10000 times the code between BENCH START and BENCH END tags (see below). Overrides any <literal>BENCH NB RUN</literal> specified in the benchmark test files.
                            </para>
                        </listitem>
                    </itemizedlist>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>exportToFile</term>
                <listitem>
                    <para>a single string</para>
                    <para>
                        File path to the result of the <function>bench_run</function> in xml format. By default, or if <literal>"", "[]"</literal> or <literal>[]</literal> is given, the output directory is <literal>TMPDIR/benchmarks/</literal>.
                    </para>
                    <para>
                        If <literal>exportToFile</literal> is a directory, creates a timestamped output file is the directory, otherwise creates the file <literal>exportToFile</literal>. If the file could not be created a warning is issued and the file is created under <literal>TMPDIR/benchmarks/</literal> instead.
                    </para>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>modutests_names</term>
                <listitem>
                    <para>a N-by-2 matrix of strings</para>
                    <para>
                        the first column lists the modules tested by <function>bench_run</function>, the second column lists the names of the benchmarks
                    </para>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>elapsed_time</term>
                <listitem>
                    <para>a vector of doubles</para>
                    <para>the execution time for each benchmark</para>
                </listitem>
            </varlistentry>
            <varlistentry>
                <term>nb_iterations</term>
                <listitem>
                    <para>a vector of doubles of size N</para>
                    <para>the number of iterations of respective test</para>
                </listitem>
            </varlistentry>
        </variablelist>
    </refsection>
    <refsection>
        <title>Description</title>
        <para>
            Performs benchmark tests, measures execution time and produces a report about benchmark tests.
        </para>
        <para>
            Searches for .tst files in benchmark test library or input parameter path under <literal>tests/benchmark</literal> subdirectory,
            executes them 10000 times and displays a report about execution time.
        </para>
        <para>
            Special tags may be inserted in the .tst file, which help to
            control the processing of the corresponding test. These tags
            are expected to be found in Scilab comments.
        </para>
        <para>These are the available tags :</para>
        <itemizedlist>
            <listitem>
                <para>
                    <literal>&lt;-- BENCH NB RUN : 10 --&gt;</literal>
                </para>
                <para>
                    By default, this test will be repeated 10 times, unless the "nb_run=###"<literal>bench_run(..)</literal> option is used. The value given for the flag can be set to any integer value.
                </para>
            </listitem>
            <listitem>
                <programlisting role="no-scilab-exec"><![CDATA[
//    <-- BENCH START -->
[code to be executed]
//    <-- BENCH END -->
]]></programlisting>
                <para>
                    Code between these tags will be repeated.
                    Any code written before/after will be executed only once
                    before/after the repetition, without being timed.
                    If these tags are missing, the entire code will be repeated.
                </para>
            </listitem>
        </itemizedlist>
    </refsection>
    <refsection>
        <title>Examples</title>
        <para>Some simple examples of invocation of bench_run</para>
        <programlisting role="example"><![CDATA[
// Launch all tests
// This may take some time...
// bench_run();
// bench_run([]);
// bench_run([],[]);

// Test one or several module
bench_run('core');
bench_run('core',[]);
bench_run(['core','string']);

// Launch one or several test in a specified module
bench_run('core',['trycatch','opcode']);

// With options
bench_run([],[],'list');
bench_run([],[],'help');
bench_run("string", [], 'nb_run=100');
// results in an output file in the current directory
bench_run("string", [], 'nb_run=100', 'my_output_file.xml');
// results in an output directory, TMPDIR/benchmarks is the default
bench_run("string", [], 'nb_run=100', TMPDIR);
]]></programlisting>
        <para>An example of a benchmark file. This file corresponds to the
            file
            SCI/modules/linear_algebra/tests/benchmarks/bench_chol.tst.
        </para>
        <programlisting role="example"><![CDATA[
// =============================================================================
// Scilab ( https://www.scilab.org/ ) - This file is part of Scilab
// Copyright (C) 2007-2008 - INRIA
//
//  This file is distributed under the same license as the Scilab package.
// =============================================================================

//==============================================================================
// Benchmark for chol function
//==============================================================================

// <-- BENCH NB RUN : 10 -->

a = 0;
b = 0;
a = rand(900, 900, 'n');
a = a'*a;

// <-- BENCH START -->
b = chol(a);
// <-- BENCH END -->
]]></programlisting>
        <para>The result of the test</para>
        <screen><![CDATA[
--> bench_run('linear_algebra','bench_chol')

           For Loop (as reference) ...........................      33.20 ms [ 1000000 x]

 001/001 - [linear_algebra] bench_chol ......................     1233.93 ms [      10 x]
]]></screen>
        <para>
        Running a subset of dedicated benchmarks by using a partial/generic testname:
        <screen><![CDATA[
--> bench_run string ascii

          For Loop (as reference) ...........................      102.98 ms [ 1000000 x]

 001/005 - [string] bench_ascii_1 ...........................      447.40 ms [   10000 x]
 002/005 - [string] bench_ascii_2 ...........................    31727.98 ms [ 1000000 x]
 003/005 - [string] bench_ascii_3 ...........................     4173.69 ms [   10000 x]
 004/005 - [string] bench_ascii_4 ...........................     5145.06 ms [   10000 x]
 005/005 - [string] bench_ascii_UTF8 ........................       23.26 ms [      10 x]
]]></screen>
        </para>
    </refsection>
    <refsection role="see also">
        <title>See also</title>
        <simplelist type="inline">
            <member>
                <link linkend="test_run">test_run</link>
            </member>
            <member>
                <link linkend="covStart">coverage</link>
            </member>
            <member>
                <link linkend="slint">slint</link>
            </member>
            <member>
                <link linkend="debug">debug</link>
            </member>
        </simplelist>
    </refsection>
    <refsection role="history">
        <title>History</title>
        <revhistory>
            <revision>
                <revnumber>6.0</revnumber>
                <revdescription>
                    <itemizedlist>
                        <listitem>
                            <literal>bench_run()</literal> can now return its results through the new
                            <literal>modutests_names</literal>, <literal>elapsed_time</literal>
                            and <literal>nb_iterations</literal> output parameters.
                        </listitem>
                        <listitem>
                            Exportation of results in XML is now possible
                        </listitem>
                        <listitem>
                            Global configuration settings mode(),
                            format(), ieee(), warning() and funcprot()
                            are now protected against tests.
                        </listitem>
                        <listitem>
                            Partial/generic test names are now allowed to run a
                            subset
                            of benchmarks dedicated to the same function/feature.
                        </listitem>
                    </itemizedlist>
                </revdescription>
            </revision>
        </revhistory>
    </refsection>
</refentry>
