Project

Profile

Help

Revision b2ef1409

Added by Norman Tovey-Walsh 5 months ago

Fix bugs in XSD test suites; add test suite report

View differences:

src/test/exceptions/xsd-extra11.xml
1
<exception>
2

  
3

  
4

  
5
  <testcase set="MS-Particles2006-07-15" group="particlesV002">
6
    <comment>Saxon runs out of memory. Still true in Saxon 9.1.</comment>
7
  </testcase>
8
  
9

  
10

  
11
</exception>
12

  
src/test/exceptions/xsd11.xml
1
<exceptions>
2

  
3

  
4
  <exception test-set="MS-Particles2006-07-15" test-case="particlesZ033_g">
5
    <reason>Until recently, Saxon ran out of memory. It now runs the test but
6
    approximates the minOccurs and maxOccurs values. I suspect this is why it decides
7
    the schema is valid although the expected result is invalid.</reason>
8
  </exception> 
9

  
10
  <exception test-set="MS-Element2006-07-15" test-case="elemT062">
11
    <reason>Bug 16161: blocked membership of substitution group</reason>
12
  </exception>
13
  <exception test-set="MS-Element2006-07-15" test-case="elemT063">
14
    <reason>Bug 16161: blocked membership of substitution group</reason>
15
  </exception>
16
  <exception test-set="MS-Element2006-07-15" test-case="elemT064">
17
    <reason>Bug 16161: blocked membership of substitution group</reason>
18
  </exception>
19
  <exception test-set="MS-Element2006-07-15" test-case="elemT065">
20
    <reason>Bug 16161: blocked membership of substitution group</reason>
21
  </exception>
22
  <exception test-set="MS-Element2006-07-15" test-case="elemT066">
23
    <reason>Bug 16161: blocked membership of substitution group</reason>
24
  </exception>
25
  <exception test-set="MS-Element2006-07-15" test-case="elemT067">
26
    <reason>Bug 16161: blocked membership of substitution group</reason>
27
  </exception>
28
  
29
  <exception test-set="MS-Additional2006-07-15" test-case="addB114">
30
    <reason>Bug 16161: blocked membership of substitution group</reason>
31
  </exception>
32
  
33
  <exception test-set="MS-Schema2006-07-15" test-case="schB8">
34
    <reason>Test assumes that http://foo/foo will not dereference. But some ISP's return a page.</reason>
35
  </exception>
36
  <exception test-set="MS-Schema2006-07-15" test-case="schE9">
37
    <reason>Test assumes that http://foo/foo will not dereference. But some ISP's return a page.</reason>
38
  </exception>
39
  
40
   <exception test-set="ElemDecl" test-case="disallowedsubst00105m">
41
     <reason>Bug 16161: blocked membership of substitution group</reason>
42
   </exception>  
43
   <exception test-set="ElemDecl" test-case="disallowedsubst00105m">
44
     <reason>Bug 16161: blocked membership of substitution group</reason>
45
   </exception>  
46
   <exception test-set="ElemDecl" test-case="disallowedsubst00106m2">
47
     <reason>Bug 16161: blocked membership of substitution group</reason>
48
   </exception>  
49
   <exception test-set="ElemDecl" test-case="disallowedsubst00501m2">
50
     <reason>Bug 16161: blocked membership of substitution group</reason>
51
   </exception>  
52
   <exception test-set="ElemDecl" test-case="disallowedsubst00502m3">
53
     <reason>Bug 16161: blocked membership of substitution group</reason>
54
   </exception>  
55
   <exception test-set="ElemDecl" test-case="disallowedsubst00502m4">
56
     <reason>Bug 16161: blocked membership of substitution group</reason>
57
   </exception>  
58
   <exception test-set="ElemDecl" test-case="disallowedsubst00503m3">
59
     <reason>Bug 16161: blocked membership of substitution group</reason>
60
   </exception>  
61
   <exception test-set="ElemDecl" test-case="disallowedsubst00503m4">
62
     <reason>Bug 16161: blocked membership of substitution group</reason>
63
   </exception>  
64
   <exception test-set="ElemDecl" test-case="disallowedsubst00503m5">
65
     <reason>Bug 16161: blocked membership of substitution group</reason>
66
   </exception>
67
   
68
   <exception test-set="Complex" test-case="complex018">
69
     <reason>Bug 16786: no content model after openContent element</reason>
70
   </exception>  
71
</exceptions>
72

  
test-suites.gradle
18 18
  // Just somewhere to hang dependencies
19 19
}
20 20

  
21
// This task will generate the test suite report from
22
// whatever results are lying around
23
task testSuiteReportsJava(
24
  type: JavaExec
25
) {
26
  def rundate = project.findProperty('rundate')
27

  
28
  inputs.files fileTree(dir: "${buildDir}/test-results")
29
  inputs.file "${projectDir}/tools/test-suites/report.xsl"
30
  outputs.file "${buildDir}/report.xml"
31
  outputs.file "${buildDir}/report.html"
32
  classpath = configurations.preprocessor
33
  workingDir = buildDir
34
  main = "net.sf.saxon.Transform"
35

  
36
  def arglist = ["-it:main",
37
                 "-xsl:${projectDir}/tools/test-suites/report.xsl",
38
                 "-ns:##any"]
39

  
40
  if (rundate != null) {
41
    arglist += ["rundate=${rundate}"]
42
  }
43
  
44
  args arglist
45
}
46

  
47
// This task *runs all the tests* and then generates the
48
// test suite report from the results
49
task runTestSuiteReportsJava(
50
  type: JavaExec,
51
  dependsOn: ["testSuitesJava"]
52
) {
53
  def rundate = project.findProperty('rundate')
54

  
55
  inputs.files fileTree(dir: "${buildDir}/test-results")
56
  inputs.file "${projectDir}/tools/test-suites/report.xsl"
57
  outputs.file "${buildDir}/report.xml"
58
  outputs.file "${buildDir}/report.html"
59
  classpath = configurations.preprocessor
60
  workingDir = buildDir
61
  main = "net.sf.saxon.Transform"
62

  
63
  def arglist = ["-it:main",
64
                 "-xsl:${projectDir}/tools/test-suites/report.xsl",
65
                 "-ns:##any"]
66

  
67
  if (rundate != null) {
68
    arglist += ["rundate=${rundate}"]
69
  }
70
  
71
  args arglist
72
}
73

  
21 74
// ============================================================
22 75

  
23 76
def langCodeMap = ["XP20": "XPath2.0",
......
42 95
    def pos = fn.toString().lastIndexOf(pathSeparator)
43 96
    def base = fn.toString().substring(pos+1)
44 97
    base = base.substring(0, base.length() - 4)
45
    task "${release}j_setup_${base}_TestResults"(
46
      type: Copy,
47
      description: "Sets up the test results directory the ${base} tests for ${RELEASE}"
48
    ) {
49
      into "${buildDir}/test-results/${release}j/${base}/${DSTAMP}"
50
      from "src/test/exceptions/${base}.xml"
51
      rename { "exceptions.xml" }
98

  
99
    def schema11 = (base == "xsd11" || base == "xsd-extra11")
100
    def schema = schema11 || (base == "xsd" || base == "xsd-extra")
101
    if (schema11 && release == "ee") {
102
      // These are a special case, they need to be renamed 'exceptions11.xml'
103
      // in the corresponding base directory. See "dependsOn" below this loop.
104
      def bdir = "xsd"
105
      if (base == "xsd-extra11") {
106
        bdir = "xsd-extra"
107
      }
108
      task "${release}j_setup_${base}_TestResults"(
109
        type: Copy,
110
        description: "Sets up the test results directory the ${base} tests for ${RELEASE}"
111
      ) {
112
        into "${buildDir}/test-results/${release}j/${bdir}/${DSTAMP}"
113
        from "src/test/exceptions/${base}.xml"
114
        rename { "exceptions11.xml" }
115
      }
116
    } else {
117
      // The schema tests only run on EE
118
      if (!schema || release == "ee") {
119
        task "${release}j_setup_${base}_TestResults"(
120
          type: Copy,
121
          description: "Sets up the test results directory the ${base} tests for ${RELEASE}"
122
        ) {
123
          into "${buildDir}/test-results/${release}j/${base}/${DSTAMP}"
124
          from "src/test/exceptions/${base}.xml"
125
          rename { "exceptions.xml" }
126
        }
127
      }
52 128
    }
53 129
  }
54 130

  
......
63 139
  }
64 140
}
65 141

  
142
tasks.findByName("eej_setup_xsd_TestResults").dependsOn "eej_setup_xsd11_TestResults"
143
tasks.findByName("eej_setup_xsd-extra_TestResults").dependsOn "eej_setup_xsd-extra11_TestResults"
144

  
66 145
task eej_setup_xsd_transformations(
67 146
  type: Copy,
68 147
  description: "Copy the transformations used by the XSD test driver"
......
474 553
        description: "Run the XSD test suite driver for ${RELEASE}/${language}/${bytecode}"
475 554
      ) {
476 555
        inputs.files fileTree(dir: "src")
477
        outputs.file "${resdir}/results_XSLT3.0_${saxonVersion}.xml"
556
        outputs.file "${resdir}/SaxonFailures${saxonVersion}.xml"
557
        outputs.file "${resdir}/SaxonResults${saxonVersion}-${language}.xml"
478 558

  
479 559
        classpath = configurations.testDrivers \
480 560
        + files("${buildDir}/classes/${release}j") \
......
482 562

  
483 563
        main = "${jpackage}.SchemaTestSuiteDriver"
484 564
        def testargs = [w3cXsdTestSuiteDir,
485
                        "catalog.xml",
486
                        "-v:${language}",
487 565
                        "-local:${resdir}",
488
                        "-lang:${language}"]
566
                        "-v:${language}"]
489 567
        if (bytecode != "") {
490 568
          testargs += [ "-bytecode:${bytecode}" ]
491 569
        }
tools/test-suites/report.xsl
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
3
                xmlns:t="http://www.w3.org/2012/08/qt-fots-results"
4
                xmlns:xs="http://www.w3.org/2001/XMLSchema"
5
                exclude-result-prefixes="xs"
6
                version="3.0">
7

  
8
  <xsl:param name="rundate" select="()"/>
9

  
10
  <xsl:variable name="input-docs-uri" as="xs:anyURI*">
11
    <xsl:for-each select="uri-collection('../../build/test-results?recurse=yes')">
12
      <!-- This stylesheet operates on the raw output, not the formatted results. -->
13
      <xsl:if test="not(contains(., '/results/'))
14
                    and not(contains(., '/xsd/'))
15
                    and not(ends-with(., '/exceptions.xml'))
16
                    and ends-with(., '.xml')">
17
        <xsl:sequence select="."/>
18
      </xsl:if>
19
    </xsl:for-each>
20
  </xsl:variable>
21

  
22
  <xsl:variable name="run-dates"
23
                select="distinct-values($input-docs-uri
24
                                        ! substring-after(., 'build/test-results/')
25
                                        ! tokenize(., '/')[3])"/>
26

  
27
  <!-- This stylesheet only expects to format the results of a single day. -->
28
  <xsl:variable name="reportdate"
29
                select="if (empty($rundate))
30
                        then sort($run-dates)[last()]
31
                        else $run-dates[. = $rundate]"/>
32

  
33
  <xsl:output indent="yes"/>
34

  
35
  <xsl:template name="xmain">
36
    <xsl:for-each select="$input-docs-uri">
37
      <xsl:message select="."/>
38
    </xsl:for-each>
39
    <xsl:sequence select="$run-dates"/>
40
    <xsl:sequence select="'R:', $reportdate"/>
41
    <doc/>
42
  </xsl:template>
43

  
44
  <xsl:template name="main">
45
    <xsl:if test="empty($reportdate)">
46
      <xsl:message terminate="yes">
47
        <xsl:text>No reports exist for </xsl:text>
48
        <xsl:sequence select="$rundate"/>
49
      </xsl:message>
50
    </xsl:if>
51

  
52
    <xsl:variable name="xmlReport" as="element()*">
53
      <xsl:call-template name="ToXml" />
54
    </xsl:variable>
55

  
56
    <xsl:result-document href="report.xml">
57
      <out>
58
        <xsl:comment>
59
          Generated - <xsl:value-of select="current-dateTime()"/>
60
        </xsl:comment>
61
        <xsl:copy-of select="$xmlReport" />
62
      </out>
63
    </xsl:result-document>
64

  
65
    <xsl:call-template name="ToHtml">
66
      <xsl:with-param name="xmlReport" select="$xmlReport" />
67
    </xsl:call-template>
68
  </xsl:template>
69

  
70
  <xsl:template name="ToXml" as="element()*">
71
      <xsl:for-each select="$input-docs-uri[contains(., '/' || $reportdate || '/')]">
72
        <xsl:variable name="input-docs" select="doc(.)"/>
73
        <xsl:variable name="test-cases"
74
                      select="$input-docs/test-suite-result/test-set/test-case"/>
75
        <xsl:variable name="failures" select="$test-cases[@result='fail']"/>
76
        <xsl:variable name="product" as="element(product)">
77
          <xsl:element name="product">
78
            <xsl:choose>
79
              <xsl:when test="exists($input-docs//product)">
80

  
81
                <xsl:choose>
82
                  <xsl:when test="contains(., 'extra')">
83
                    <xsl:attribute name="test-suite" select="'saxon-extra'" />
84
                  </xsl:when>
85
                  <xsl:when test="not(contains(., 'extra'))">
86
                    <xsl:attribute name="test-suite" select="'w3c'" />
87
                  </xsl:when>
88
                </xsl:choose>
89

  
90
                <xsl:attribute name="name" select="$input-docs//product/@name" />
91
                <xsl:attribute name="version" select="$input-docs//product/@version" />
92
                <xsl:attribute name="bytecode" select="$input-docs//product/@bytecode" />
93
                <xsl:attribute name="language" select="$input-docs//product/@language" />
94
                <xsl:attribute name="date-run" select="$input-docs//test-run/@date-run"/>
95

  
96
              </xsl:when>
97
              <xsl:when test="exists($input-docs//implementation)">
98

  
99
                <xsl:choose>
100
                  <xsl:when test="contains(., 'extra')">
101
                    <xsl:attribute name="test-suite" select="'saxon-extra'" />
102
                  </xsl:when>
103
                  <xsl:when test="not(contains(., 'extra'))">
104
                    <xsl:attribute name="test-suite" select="'w3c'" />
105
                  </xsl:when>
106
                </xsl:choose>
107

  
108
                <xsl:attribute name="name" select="$input-docs//implementation/@name" />
109
                <xsl:attribute name="version" select="$input-docs//implementation/@version" />
110
                <xsl:attribute name="bytecode" select="$input-docs//implementation/configuration/@bytecode" />
111
                <xsl:attribute name="language" select="$input-docs//implementation/configuration/@lang" />
112
                <xsl:attribute name="jit" select="$input-docs//implementation/configuration/@jit" />
113
                <xsl:attribute name="date-run" select="$input-docs//test-run/@dateRun"/>
114

  
115
              </xsl:when>
116
              <xsl:otherwise>
117
                <xsl:message>UNKNOWN Found with uri=<xsl:value-of select="."/></xsl:message>
118
                <xsl:attribute name="name" select="'UNKNOWN'" />
119
                <xsl:attribute name="bytecode" select="'NA'" />
120
                <xsl:attribute name="langauge" select="'NA'" />
121

  
122
              </xsl:otherwise>
123
            </xsl:choose>
124
          </xsl:element>
125
        </xsl:variable>
126

  
127
        <product edition="{$product/@name}" bytecode="{$product/@bytecode}"
128
                 lang="{$product/@language}" version="{$product/@version}">
129
          <xsl:if test="exists($product/@jit)">
130
            <xsl:attribute name="jit" select="$product/@jit"/>
131
          </xsl:if>
132
          <xsl:if test="contains(., 'unfolded')">
133
            <xsl:attribute name="unfolded" select="'yes'"/>
134
          </xsl:if>
135
          <test-run date="{$product/@date-run}" test-suite="{$product/@test-suite}"/>
136
          <tests successes="{count($test-cases[@result='pass'])}"
137
                 failures="{count($failures)}"
138
                 incorrect-errorcode="{count($test-cases[@result='wrongError'])}"
139
                 not-run="{count($test-cases[@result='n/a'])}"/>
140
          <xsl:if test="count($failures) gt 30">
141
            <failure-distribution>
142
              <xsl:for-each-group select="$failures" group-by="../@name">
143
                <test-set-failures name="{current-grouping-key()}"
144
                                   no="{count(current-group())}"/>
145
              </xsl:for-each-group>
146
            </failure-distribution>
147
          </xsl:if>
148
          <xsl:for-each select="$failures">
149
            <test-case-fail name="{@name}" comment="{@comment}"/>
150
          </xsl:for-each>
151
        </product>
152
      </xsl:for-each>
153
  </xsl:template>
154

  
155
  <xsl:template name="ToHtml">
156
    <xsl:param name="xmlReport" as="element()*" />
157
    <xsl:result-document href="report.html">
158
      <html>
159
        <head><title>Saxon Release reports</title>
160

  
161
        <style type="text/css">
162
          .tg  {border-collapse:collapse;border-spacing:0;}
163
          .tg td{border-color:black;border-style:solid;border-width:1px;font-family:Arial, sans-serif;font-size:14px;
164
          overflow:hidden;padding:10px 5px;word-break:normal;}
165
          .tg th{border-color:black;border-style:solid;border-width:1px;font-family:Arial, sans-serif;font-size:14px;
166
          font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;}
167
          .tg .tg-1wig{font-weight:bold;text-align:left;vertical-align:top}
168
          .tg .tg-mnhx{background-color:#fe0000;text-align:left;vertical-align:top}
169
          .tg .tg-fd62{background-color:#32cb00;text-align:left;vertical-align:top}
170
          .tg .tg-s7ni{background-color:#f8ff00;text-align:left;vertical-align:top}
171
          .tg .tg-0lax{text-align:left;vertical-align:top}
172
        </style>
173
        </head>
174
        <body>
175
          <h1>Saxon Release Reports</h1>
176
          <p>
177
            <b>Saxon Version: <xsl:value-of select="string(($xmlReport[1])/@version)"/></b>
178
          </p>
179
          <p>Date: <xsl:value-of select="$reportdate"/></p>
180

  
181
          <table class="tg">
182
            <thead>
183
              <th class="tg-1wig">Product</th>
184
              <th class="tg-fd62"><b>Pass</b></th>
185
              <th class="tg-mnhx"><b>Fail</b></th>
186
              <th class="tg-s7ni"><b>Incorrect Error code</b></th>
187
              <th class="tg-0lax"><b>Skip</b></th>
188
              <th class="tg-0lax"><b>Test Failures</b></th>
189
            </thead>
190
            <!--Count=<xsl:value-of select="count($input-docs)"/> -->
191
            <xsl:for-each select="$xmlReport">
192
              <xsl:sort select="@edition" />
193
              <xsl:sort select="@lang" />
194
              <!-- Dir=<xsl:value-of select="string(document-uri(.))"/> -->
195

  
196
              <xsl:element name="tr">
197
                <td  class="tg-1wig" width="50">
198
                  <xsl:value-of select="@edition"/>
199
                  <br/>
200
                  Bytecode=<xsl:value-of select="@bytecode"/>
201
                  <br/>
202
                  <xsl:if test="exists(@jit)">
203
                    JIT=<xsl:value-of select="@jit"/>
204
                    <br/>
205
                  </xsl:if>
206

  
207
                  Lang=<xsl:value-of select="@lang"/>
208
                  <br/>
209
                  Test-Suite= <xsl:value-of select="./test-run/@test-suite"/>
210
                  <br/>
211
                  <xsl:if test="exists(@export)">
212
                    Export= <xsl:value-of select="@export"/>
213
                    <br/>
214
                  </xsl:if>
215

  
216
                  <!--Dir=<xsl:value-of select="../@dir"/> -->
217

  
218
                </td>
219
                <td  class="tg-fd62">
220
                  <xsl:value-of select="tests/@successes"/>
221
                </td>
222
                <td class="tg-mnhx">
223
                  <xsl:value-of select="tests/@failures"/>
224
                </td>
225
                <td class="tg-s7ni">
226
                  <xsl:value-of select="tests/@incorrect-errorcode"/>
227
                </td>
228
                <td class="tg-0lax">
229
                  <xsl:value-of select="tests/@not-run"/>
230
                </td>
231
                <td class="tg-0lax">
232
                  <xsl:for-each select="test-case-fail">
233
                    <xsl:value-of select="@name"/>, &#x20;
234

  
235
                  </xsl:for-each>
236
                </td>
237

  
238

  
239

  
240
              </xsl:element>
241

  
242

  
243

  
244

  
245
            </xsl:for-each>
246
          </table>
247
        </body>
248

  
249
      </html>
250
    </xsl:result-document>
251

  
252
  </xsl:template>
253

  
254

  
255
</xsl:stylesheet>

Also available in: Unified diff