Project

Profile

Help

How to connect?
Download (20 KB) Statistics
| Branch: | Tag: | Revision:

he / latest9.9 / samples / cs / TestRunner9.9 / TestDriver.cs @ 063bf4dc

1
using System;
2
using System.Collections;
3
using System.Collections.Generic;
4
using System.Text;
5
using Saxon.Api;
6
using System.Xml;
7
using System.IO;
8
using System.Text.RegularExpressions;
9
using System.Globalization;
10
using TestRunner;
11

    
12

    
13
//using net.sf.saxon.Version;
14
using JFeatureKeys = net.sf.saxon.lib.FeatureKeys;
15
using javax.xml.transform.stream;
16
using System.Diagnostics;
17

    
18
namespace TestRunner
19
{
20

    
21

    
22

    
23
    public abstract class TestDriver {
24

    
25
        protected string resultsDir = null;
26
        public ResultsDocument resultsDoc;
27
        protected int successes = 0;
28
        protected int total = 200000;
29
        protected int failures = 0;
30
        protected int notrun = 0;
31
        protected int wrongErrorResults = 0;
32
        protected bool unfolded = false;
33
        protected bool saveResults = false;
34
        protected int generateByteCode = 0;
35
        protected TreeModel treeModel = TreeModel.TinyTree;
36
        protected bool debug = false;
37
        protected Regex testPattern = null;
38
        protected string requestedTestSet = null;
39
        protected string testSuiteDir;
40
        protected Processor driverProc = new Processor(true);
41
        protected Serializer driverSerializer;
42
        protected Spec spec;
43
        protected Dictionary<string, XdmNode> exceptionsMap = new Dictionary<string, XdmNode>();
44
        protected Dictionary<string, Environment> globalEnvironments = new Dictionary<string, Environment>();
45
        protected Dictionary<string, Environment> localEnvironments = new Dictionary<string, Environment>();
46
        protected IFeedbackListener feedback = new DefaultFeedbackListener();
47
        protected bool useXslt30Transformer = true;  // Temporary for controlling test processor
48

    
49

    
50
        public abstract string catalogNamespace();
51

    
52
        public int GenerateByteCode {
53
            get { return generateByteCode; }
54
            set { generateByteCode = value; }
55
        }
56

    
57
        public class DefaultFeedbackListener : IFeedbackListener
58
        {
59

    
60
            public void Feedback(int passed, int failed, int total)
61
            {
62
                Console.WriteLine("Done " + (passed + failed) + " of " + total);
63
                Trace.WriteLine("Done " + (passed + failed) + " of " + total);
64

    
65
            }
66

    
67

    
68
            public void Message(String message, bool popup)
69
            {
70
                try {
71
                    Console.WriteLine(message);
72
                    Trace.WriteLine(message);
73
                } catch (Exception e) {
74
                    Console.WriteLine("issue with the message");
75

    
76
                }
77

    
78
            }
79
        }
80

    
81

    
82
        public Spec Spec {
83
            get { return spec; }
84
        }
85
        public void setFeedbackListener(IFeedbackListener f)
86
        {
87
            feedback = f;
88
        }
89

    
90
        public int Failures
91
        {
92
            get { return failures; }
93
            set { failures = value; }
94
        }
95

    
96
        public bool Unfolded
97
        {
98
            get { return unfolded; }
99
            set { unfolded = value; }
100
        }
101

    
102
        public TreeModel TreeModel
103
        {
104
            get { return treeModel; }
105
            set { treeModel = value; }
106
        }
107
        public void go(string[] args) {
108

    
109
            //    AutoActivate.activate(driverProc);
110
            driverSerializer = driverProc.NewSerializer();
111
            //testSuiteDir = args[0];
112

    
113
            Trace.Listeners.Add(new TextWriterTraceListener(Console.Out));
114

    
115
            testSuiteDir = args[0];
116
            if (testSuiteDir.EndsWith("/"))
117
            {
118
                testSuiteDir = testSuiteDir.Substring(0, testSuiteDir.Length - 1);
119
            }
120

    
121

    
122
            resultsDir = args[1];
123
            if (resultsDir.EndsWith("/"))
124
            {
125
                resultsDir = resultsDir.Substring(0, resultsDir.Length - 1);
126
            }
127
            string catalog = testSuiteDir + "/catalog.xml";
128
            string specStr = null;
129

    
130
            for (int i = 2; i < args.Length; i++) {
131
                if (args[i].StartsWith("-t:")) {
132
                    testPattern = new Regex(args[i].Substring(3));
133
                }
134
                if (args[i].StartsWith("-s:")) {
135
                    requestedTestSet = args[i].Substring(3);
136
                }
137
                if (args[i].StartsWith("-debug")) {
138
                    debug = true;
139
                }
140
                if (args[i].Equals("-unfolded")) {
141
                    unfolded = true;
142
                }
143
                if (args[i].Equals("-save"))
144
                {
145
                    saveResults = true;
146
                }
147
                if (args[i].StartsWith("-bytecode"))
148
                {
149
                    if (args[i].Substring(10).Equals("on"))
150
                    {
151
                        generateByteCode = 1;
152
                    }
153
                    else if (args[i].Substring(10).Equals("debug"))
154
                    {
155
                        generateByteCode = 2;
156
                    }
157
                    else
158
                    {
159
                        generateByteCode = 0;
160
                    }
161
                }
162
                /* if (args[i].StartsWith("-tree"))
163
                 {
164
                     if (args[i].Substring(6).EqualsIgnoreCase("dom"))
165
                     {
166
                         treeModel = new DOMObjectModel();
167
                     }
168
                     else if (args[i].Substring(6).EqualsIgnoreCase("jdom"))
169
                     {
170
                         treeModel = new JDOMObjectModel();
171
                     }
172
                     else if (args[i].Substring(6).EqualsIgnoreCase("jdom2"))
173
                     {
174
                         treeModel = new JDOM2ObjectModel();
175
                     }
176
                     else if (args[i].Substring(6).EqualsIgnoreCase("dom4j"))
177
                     {
178
                         treeModel = new DOM4JObjectModel();
179
                     }
180
                     else if (args[i].Substring(6).EqualsIgnoreCase("xom"))
181
                     {
182
                         treeModel = new XOMObjectModel();
183
                     }
184
                     else if (args[i].Substring(6).EqualsIgnoreCase("axiom"))
185
                     {
186
                         treeModel = new AxiomObjectModel();
187
                     }
188
                     else if (args[i].Substring(6).EqualsIgnoreCase("tinytree"))
189
                     {
190
                         treeModel = TreeModel.TINY_TREE;
191
                     }
192
                     else if (args[i].Substring(6).EqualsIgnoreCase("condensed"))
193
                     {
194
                         treeModel = TreeModel.TINY_TREE_CONDENSED;
195
                     }
196
                     else if (args[i].Substring(6).EqualsIgnoreCase("linked"))
197
                     {
198
                         treeModel = TreeModel.LINKED_TREE;
199
                     }
200
                     else
201
                     {
202
                         throw new Exception("The TreeModel specified does not exist");
203
                     }
204
                 }*/
205
                if (args[i].StartsWith("-lang"))
206
                {
207
                    specStr = args[i].Substring(6);
208
                    processSpec(specStr);
209
                }
210

    
211
                // Temporary for controlling test processor
212
                if (args[i].StartsWith("-xt30")) {
213
                    if (args[i].Substring(6).Equals("on")) {
214
                        useXslt30Transformer = true;
215
                    } else if (args[i].Substring(6).Equals("off")) {
216
                        useXslt30Transformer = false;
217
                    }
218
                }
219
            }
220
            if (resultsDir == null)
221
            {
222
                feedback.Message("No results directory specified (use -o:dirname)", true);
223
                /*if (guiForm == null)
224
                {
225
                    System.exit(2);
226
                }*/
227
            }
228
            if (resultsDoc == null)
229
            {
230
                feedback.Message("No results document specified (use -lang option)", true);
231
                /*if (guiForm == null)
232
                {
233
                    System.exit(2);
234
                }*/
235
            }
236

    
237
            driverSerializer.SetOutputStream(System.Console.OpenStandardError());
238
            driverSerializer.SetOutputProperty(Serializer.OMIT_XML_DECLARATION, "yes");
239
            processCatalog(catalog);
240
            printResults(resultsDir + "/results" + driverProc.ProductVersion + ".xml");
241
        }
242

    
243
        public string getResultsDir() {
244
            return resultsDir;
245
        }
246

    
247
        public abstract void processSpec(string specStr);
248

    
249
        internal class MyLogger : net.sf.saxon.lib.Logger
250
        {
251
            StreamWriter writer;
252

    
253
            public MyLogger() {
254
               // writer =File.CreateText("logger.txt");
255
               
256
            }
257
            public override StreamResult asStreamResult()
258
            {
259
                return null;
260
            }
261

    
262
            public override void println(string str, int i)
263
            {
264
                Trace.WriteLine(str);
265
             //  writer.WriteLine(str);
266
            }
267
        }
268

    
269

    
270
        private void processCatalog(String catalogFile) {
271
       if (generateByteCode == 1) {
272
            driverProc.SetProperty(JFeatureKeys.GENERATE_BYTE_CODE, "true");
273
            driverProc.SetProperty(JFeatureKeys.DEBUG_BYTE_CODE, "false");
274
        } else if (generateByteCode == 2) {
275
            driverProc.SetProperty(JFeatureKeys.GENERATE_BYTE_CODE, "true");
276
            driverProc.SetProperty(JFeatureKeys.DEBUG_BYTE_CODE, "true");
277
        } else {
278
            driverProc.SetProperty(JFeatureKeys.GENERATE_BYTE_CODE, "false");
279
            driverProc.SetProperty(JFeatureKeys.DEBUG_BYTE_CODE, "false");
280
        }
281
         
282
        DocumentBuilder catbuilder = driverProc.NewDocumentBuilder();
283
        catbuilder.TreeModel = treeModel;
284
        XdmNode catalog = catbuilder.Build(new Uri(catalogFile)); 
285
        XPathCompiler xpc = driverProc.NewXPathCompiler();
286
        //xpc.XPathLanguageVersion = "3.1";
287
        xpc.Caching = true;
288
        xpc.DeclareNamespace("", catalogNamespace());
289

    
290
        createGlobalEnvironments(catalog, xpc);
291

    
292
        try {
293
            writeResultFilePreamble(driverProc, catalog);
294
        } catch (Exception e) {
295
            System.Console.WriteLine(e.Message);
296
        }
297

    
298
        readExceptionsFile();
299

    
300

    
301
        if (requestedTestSet != null) {
302
            try {
303
                XdmNode funcSetNode = (XdmNode) xpc.EvaluateSingle("//test-set[@name='" + requestedTestSet + "']", catalog);
304
                if (funcSetNode == null) {
305
                    throw new Exception("Test-set " + requestedTestSet + " not found!");
306
                }
307
                processTestSet(catbuilder, xpc, funcSetNode);
308
            } catch (Exception e1) {
309
                //e1.printStackTrace();
310
                System.Console.WriteLine(e1.Message);
311
            }
312
        } else {
313
            foreach (XdmItem testSet in xpc.Evaluate("//test-set", catalog)) {
314
                processTestSet(catbuilder, xpc, ((XdmNode) testSet.Simplify));
315
            }
316
        }
317
        try {
318
            writeResultFilePostamble();
319
        } catch (Exception e) {
320
            System.Console.WriteLine(e.Message);
321
            //e.printStackTrace();
322
        }
323

    
324

    
325
    }
326

    
327
    /**
328
     * Look for an exceptions.xml document with the general format:
329
     * <p/>
330
     * <exceptions xmlns="...test catalog namespace...">
331
     * <exception test-set ="testset1" test-case="testcase" run="yes/no/not-unfolded"
332
     * bug="bug-reference" reason="">
333
     * <results>
334
     * ... alternative expected results ...
335
     * </results>
336
     * <optimization>
337
     * ... assertions about the "explain" tree
338
     * </optimization>
339
     * </exception>
340
     * </exceptions>
341
     */
342

    
343
    protected void readExceptionsFile() {
344

    
345
        XdmNode exceptionsDoc = null;
346
        DocumentBuilder exceptBuilder = driverProc.NewDocumentBuilder();
347
        QName testCase = new QName("", "test-case");
348
        try {
349
            exceptionsDoc = exceptBuilder.Build(new Uri(resultsDir + "/exceptions.xml"));
350
            IEnumerator iter = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, new QName("", "exception"));
351
            while (iter.Current != null) {
352
                XdmNode entry = (XdmNode) iter.Current;
353
                string test = entry.GetAttributeValue(testCase);
354
                if (test != null) {
355
                    exceptionsMap.Add(test, entry);
356
                }
357
                iter.MoveNext();
358
            }
359
        } catch (Exception e) {
360
            feedback.Message("*** Failed to process exceptions file: "+ e.Message, true);
361
            //printError("*** Failed to process exceptions file: ", e.getMessage()); //TODO - review this code later
362
        }
363

    
364
    }
365

    
366
    protected abstract void createGlobalEnvironments(
367
            XdmNode catalog, XPathCompiler xpc);
368

    
369
    protected void createLocalEnvironments(XdmNode testSetDocNode) {
370
        localEnvironments.Clear();
371
        Environment defaultEnvironment =
372
				Environment.createLocalEnvironment(testSetDocNode.BaseUri, generateByteCode, unfolded, spec);
373
        localEnvironments.Add("default", defaultEnvironment);
374
    }
375

    
376
    protected Environment getEnvironment(XdmNode testCase, XPathCompiler xpc) {
377
        string testCaseName = testCase.GetAttributeValue(new QName("name"));
378
        XdmNode environmentNode = (XdmNode) xpc.EvaluateSingle("environment", testCase);
379
        Environment env;
380
        if (environmentNode == null) {
381
            env = localEnvironments["default"];
382
        } else {
383
            string envName = environmentNode.GetAttributeValue(new QName("ref"));
384
            if (envName == null || envName.Equals("")) {
385
                env = Environment.processEnvironment(this, xpc, environmentNode, null, localEnvironments["default"]);
386
                bool baseUriCheck = ((XdmAtomicValue) xpc.EvaluateSingle("static-base-uri/@uri='#UNDEFINED'", environmentNode)).GetBooleanValue();
387
                if (baseUriCheck) {
388
                    //writeTestcaseElement(testCaseName, "notRun", "static-base-uri not supported", null);
389
                    return null;
390
                }
391
            } else {
392
                try
393
                {
394
                    env = localEnvironments[envName];
395
                }catch(Exception){
396
                    env = null;
397
                }
398
                if (env == null) {
399
                    try
400
                    {
401
                        env = globalEnvironments[envName];
402
                    }
403
                    catch (Exception e) { }
404
                }
405
                if (env == null) {
406
                    foreach (XdmItem e in xpc.Evaluate("//environment[@name='" + envName + "']", testCase)) {
407
                        Environment.processEnvironment(this, xpc, e, localEnvironments, localEnvironments["default"]);
408
                    }
409
                    try
410
                    {
411
                        env = localEnvironments[envName];
412
                    } catch(Exception e) {}
413
                }
414
                if (env == null) {
415
                    System.Console.WriteLine("*** Unknown environment " + envName);
416
                    //println("*** Unknown environment " + envName);
417
                    //writeTestcaseElement(testCaseName, "fail", "Environment " + envName + " not found", null);
418
                    failures++;
419
                    return null;
420
                }
421

    
422
            }
423
        }
424
        return env;
425
    }
426

    
427
    public void writeResultFilePreamble(Processor processor, XdmNode catalog) {
428
        resultsDoc.writeResultFilePreamble(processor, catalog);
429
    }
430

    
431
    public void writeResultFilePostamble(){
432
        resultsDoc.writeResultFilePostamble();
433
    }
434

    
435
    public void startTestSetElement(XdmNode testSetNode) {
436
        resultsDoc.startTestSetElement(testSetNode);
437
    }
438

    
439
    public void writeTestSetEndElement() {
440
        resultsDoc.endElement();
441
    }
442

    
443

    
444
    private void processTestSet(DocumentBuilder catbuilder, XPathCompiler xpc, XdmNode testSetNode) {
445
        string testName;
446
        string testSet;
447
        startTestSetElement(testSetNode);
448
        Uri testSetFile = new Uri(testSuiteDir + "/" + testSetNode.GetAttributeValue(new QName("file")));
449
        XdmNode testSetDocNode = catbuilder.Build(testSetFile);
450
        createLocalEnvironments(testSetDocNode);
451
        bool run = true;
452
        // TODO: this won't pick up any test-set level dependencies in the XSLT 3.0 catalog
453
        if (((XdmAtomicValue) xpc.Evaluate("exists(/test-set/dependency)", testSetDocNode).Simplify).GetBooleanValue()) {
454
            foreach (XdmItem dependency in xpc.Evaluate("/test-set/dependency", testSetDocNode)) {
455
                if (!dependencyIsSatisfied((XdmNode) dependency, localEnvironments["default"])) {
456
                    run = false;
457
                }
458
            }
459
        }
460
        if (run) {
461
            if (testPattern == null) {
462
                foreach (XdmItem env in xpc.Evaluate("//environment[@name]", testSetDocNode)) {
463
                    Environment.processEnvironment(this, xpc, env, localEnvironments, localEnvironments["default"]);
464
                }
465
            }
466
            testSet = xpc.EvaluateSingle("/test-set/@name", testSetDocNode).GetStringValue();
467
            foreach (XdmItem testCase in xpc.Evaluate("//test-case", testSetDocNode)) {
468

    
469
                testName = xpc.EvaluateSingle("@name", testCase).GetStringValue();
470

    
471
                ////
472
                //if (testName.Contains("base64-909"))
473
                //{
474
                //    int num = 0;
475
                  //  System.Console.WriteLine("Test driver");
476

    
477
                //}
478

    
479
                ///
480
                if (testPattern != null && !testPattern.IsMatch(testName)) {
481
                    continue;
482
                }
483
                println("-s:" + testSet + " -t:" + testName);
484

    
485
                runTestCase((XdmNode) testCase, xpc);
486
            }
487
        }
488
        writeTestSetEndElement();
489
    }
490

    
491
    protected abstract void runTestCase(XdmNode testCase, XPathCompiler catalogXpc);
492

    
493
    //public void setTestDriverForm(TestDriverForm gui) {
494
    //    guiForm = gui;
495
    //}
496

    
497
    public void println(string data)
498
    {
499
        //if (guiForm != null)
500
        //{
501
            //guiForm.setResultTextArea(data);
502
        //}
503
        //else
504
        {
505
            feedback.Message(data+"\n", false);
506
        }
507
    }
508

    
509
    public void printResults(string resultsFileStr)
510
    {
511
       /* if (guiForm != null)
512
        {
513
            guiForm.printResults("Result: " + successes + " successes, " + failures + " failures, " + wrongErrorResults + " incorrect ErrorCode, " + notrun + " not run", resultsFileStr, resultsDir);
514
        }
515
        else
516
        {*/
517
        feedback.Message(successes + " successes, " + failures + " failures, " + wrongErrorResults + " incorrect ErrorCode, " + notrun + " not run", false);
518
        //}
519
    }
520

    
521
    public void printError(string error, string message)
522
    {
523
        /*if (guiForm != null)
524
        {
525
            guiForm.errorPopup(error);
526
            System.err.println(error + message);
527
        }
528
        else
529
        {*/
530
        feedback.Message(error + message, true);
531
        //}
532
    }
533

    
534
    public abstract bool dependencyIsSatisfied(XdmNode dependency, Environment env);
535

    
536
}
537

    
538
//
539
// The contents of this file are subject to the Mozilla Public License Version 1.0 (the "License");
540
// you may not use this file except in compliance with the License. You may obtain a copy of the
541
// License at http://www.mozilla.org/MPL/
542
//
543
// Software distributed under the License is distributed on an "AS IS" basis,
544
// WITHOUT WARRANTY OF ANY KIND, either express or implied.
545
// See the License for the specific language governing rights and limitations under the License.
546
//
547
// The Original Code is: all this file.
548
//
549
// The Initial Developer of the Original Code is Saxonica Limited
550
//
551
// Portions created by (your name) are Copyright (C) (your legal entity). All Rights Reserved.
552
//
553
// Contributor(s): none.
554
//
555

    
556
}
(10-10/19)