Commit | Line | Data |
b965d173 |
1 | #!/usr/bin/perl -w |
2 | |
3 | BEGIN { |
4 | if ($ENV{PERL_CORE}) { |
5 | # FIXME |
6 | print "1..0 # Skip until we figure out why it exists with no output just after the plan\n"; |
7 | exit 0; |
8 | } |
9 | } |
10 | |
11 | use strict; |
12 | use lib 't/lib'; |
13 | |
14 | use Test::More; |
15 | |
16 | use File::Spec; |
17 | |
18 | use Test::Harness qw(execute_tests); |
19 | |
20 | # unset this global when self-testing ('testcover' and etc issue) |
21 | local $ENV{HARNESS_PERL_SWITCHES}; |
22 | |
23 | { |
24 | |
25 | # if the harness wants to save the resulting TAP we shouldn't |
26 | # do it for our internal calls |
27 | local $ENV{PERL_TEST_HARNESS_DUMP_TAP} = 0; |
28 | |
29 | my $TEST_DIR = 't/sample-tests'; |
30 | my $PER_LOOP = 4; |
31 | |
32 | my $results = { |
33 | 'descriptive' => { |
34 | 'failed' => {}, |
35 | 'todo' => {}, |
36 | 'totals' => { |
37 | 'bad' => 0, |
38 | 'bonus' => 0, |
39 | 'files' => 1, |
40 | 'good' => 1, |
41 | 'max' => 5, |
42 | 'ok' => 5, |
43 | 'skipped' => 0, |
44 | 'sub_skipped' => 0, |
45 | 'tests' => 1, |
46 | 'todo' => 0 |
47 | } |
48 | }, |
49 | join( |
50 | ',', qw( |
51 | descriptive die die_head_end die_last_minute duplicates |
52 | head_end head_fail inc_taint junk_before_plan lone_not_bug |
53 | no_nums no_output schwern sequence_misparse shbang_misparse |
54 | simple simple_fail skip skip_nomsg skipall skipall_nomsg |
55 | stdout_stderr switches taint todo_inline |
56 | todo_misparse too_many vms_nit |
57 | ) |
58 | ) => { |
59 | 'failed' => { |
60 | 't/sample-tests/die' => { |
61 | 'canon' => '??', |
62 | 'estat' => 1, |
63 | 'failed' => '??', |
64 | 'max' => '??', |
65 | 'name' => 't/sample-tests/die', |
66 | 'wstat' => '256' |
67 | }, |
68 | 't/sample-tests/die_head_end' => { |
69 | 'canon' => '??', |
70 | 'estat' => 1, |
71 | 'failed' => '??', |
72 | 'max' => '??', |
73 | 'name' => 't/sample-tests/die_head_end', |
74 | 'wstat' => '256' |
75 | }, |
76 | 't/sample-tests/die_last_minute' => { |
77 | 'canon' => '??', |
78 | 'estat' => 1, |
79 | 'failed' => 0, |
80 | 'max' => 4, |
81 | 'name' => 't/sample-tests/die_last_minute', |
82 | 'wstat' => '256' |
83 | }, |
84 | 't/sample-tests/duplicates' => { |
85 | 'canon' => '??', |
86 | 'estat' => '', |
87 | 'failed' => '??', |
88 | 'max' => 10, |
89 | 'name' => 't/sample-tests/duplicates', |
90 | 'wstat' => '' |
91 | }, |
92 | 't/sample-tests/head_fail' => { |
93 | 'canon' => 2, |
94 | 'estat' => '', |
95 | 'failed' => 1, |
96 | 'max' => 4, |
97 | 'name' => 't/sample-tests/head_fail', |
98 | 'wstat' => '' |
99 | }, |
100 | 't/sample-tests/inc_taint' => { |
101 | 'canon' => 1, |
102 | 'estat' => 1, |
103 | 'failed' => 1, |
104 | 'max' => 1, |
105 | 'name' => 't/sample-tests/inc_taint', |
106 | 'wstat' => '256' |
107 | }, |
108 | 't/sample-tests/no_nums' => { |
109 | 'canon' => 3, |
110 | 'estat' => '', |
111 | 'failed' => 1, |
112 | 'max' => 5, |
113 | 'name' => 't/sample-tests/no_nums', |
114 | 'wstat' => '' |
115 | }, |
116 | 't/sample-tests/no_output' => { |
117 | 'canon' => '??', |
118 | 'estat' => '', |
119 | 'failed' => '??', |
120 | 'max' => '??', |
121 | 'name' => 't/sample-tests/no_output', |
122 | 'wstat' => '' |
123 | }, |
124 | 't/sample-tests/simple_fail' => { |
125 | 'canon' => '2 5', |
126 | 'estat' => '', |
127 | 'failed' => 2, |
128 | 'max' => 5, |
129 | 'name' => 't/sample-tests/simple_fail', |
130 | 'wstat' => '' |
131 | }, |
132 | 't/sample-tests/switches' => { |
133 | 'canon' => 1, |
134 | 'estat' => '', |
135 | 'failed' => 1, |
136 | 'max' => 1, |
137 | 'name' => 't/sample-tests/switches', |
138 | 'wstat' => '' |
139 | }, |
140 | 't/sample-tests/todo_misparse' => { |
141 | 'canon' => 1, |
142 | 'estat' => '', |
143 | 'failed' => 1, |
144 | 'max' => 1, |
145 | 'name' => 't/sample-tests/todo_misparse', |
146 | 'wstat' => '' |
147 | }, |
148 | 't/sample-tests/too_many' => { |
149 | 'canon' => '4-7', |
150 | 'estat' => 4, |
151 | 'failed' => 4, |
152 | 'max' => 3, |
153 | 'name' => 't/sample-tests/too_many', |
154 | 'wstat' => '1024' |
155 | }, |
156 | 't/sample-tests/vms_nit' => { |
157 | 'canon' => 1, |
158 | 'estat' => '', |
159 | 'failed' => 1, |
160 | 'max' => 2, |
161 | 'name' => 't/sample-tests/vms_nit', |
162 | 'wstat' => '' |
163 | } |
164 | }, |
165 | 'todo' => { |
166 | 't/sample-tests/todo_inline' => { |
167 | 'canon' => 2, |
168 | 'estat' => '', |
169 | 'failed' => 1, |
170 | 'max' => 2, |
171 | 'name' => 't/sample-tests/todo_inline', |
172 | 'wstat' => '' |
173 | } |
174 | }, |
175 | 'totals' => { |
176 | 'bad' => 13, |
177 | 'bonus' => 1, |
178 | 'files' => 28, |
179 | 'good' => 15, |
180 | 'max' => 77, |
181 | 'ok' => 78, |
182 | 'skipped' => 2, |
183 | 'sub_skipped' => 2, |
184 | 'tests' => 28, |
185 | 'todo' => 2 |
186 | } |
187 | }, |
188 | 'die' => { |
189 | 'failed' => { |
190 | 't/sample-tests/die' => { |
191 | 'canon' => '??', |
192 | 'estat' => 1, |
193 | 'failed' => '??', |
194 | 'max' => '??', |
195 | 'name' => 't/sample-tests/die', |
196 | 'wstat' => '256' |
197 | } |
198 | }, |
199 | 'todo' => {}, |
200 | 'totals' => { |
201 | 'bad' => 1, |
202 | 'bonus' => 0, |
203 | 'files' => 1, |
204 | 'good' => 0, |
205 | 'max' => 0, |
206 | 'ok' => 0, |
207 | 'skipped' => 0, |
208 | 'sub_skipped' => 0, |
209 | 'tests' => 1, |
210 | 'todo' => 0 |
211 | } |
212 | }, |
213 | 'die_head_end' => { |
214 | 'failed' => { |
215 | 't/sample-tests/die_head_end' => { |
216 | 'canon' => '??', |
217 | 'estat' => 1, |
218 | 'failed' => '??', |
219 | 'max' => '??', |
220 | 'name' => 't/sample-tests/die_head_end', |
221 | 'wstat' => '256' |
222 | } |
223 | }, |
224 | 'todo' => {}, |
225 | 'totals' => { |
226 | 'bad' => 1, |
227 | 'bonus' => 0, |
228 | 'files' => 1, |
229 | 'good' => 0, |
230 | 'max' => 0, |
231 | 'ok' => 4, |
232 | 'skipped' => 0, |
233 | 'sub_skipped' => 0, |
234 | 'tests' => 1, |
235 | 'todo' => 0 |
236 | } |
237 | }, |
238 | 'die_last_minute' => { |
239 | 'failed' => { |
240 | 't/sample-tests/die_last_minute' => { |
241 | 'canon' => '??', |
242 | 'estat' => 1, |
243 | 'failed' => 0, |
244 | 'max' => 4, |
245 | 'name' => 't/sample-tests/die_last_minute', |
246 | 'wstat' => '256' |
247 | } |
248 | }, |
249 | 'todo' => {}, |
250 | 'totals' => { |
251 | 'bad' => 1, |
252 | 'bonus' => 0, |
253 | 'files' => 1, |
254 | 'good' => 0, |
255 | 'max' => 4, |
256 | 'ok' => 4, |
257 | 'skipped' => 0, |
258 | 'sub_skipped' => 0, |
259 | 'tests' => 1, |
260 | 'todo' => 0 |
261 | } |
262 | }, |
263 | 'duplicates' => { |
264 | 'failed' => { |
265 | 't/sample-tests/duplicates' => { |
266 | 'canon' => '??', |
267 | 'estat' => '', |
268 | 'failed' => '??', |
269 | 'max' => 10, |
270 | 'name' => 't/sample-tests/duplicates', |
271 | 'wstat' => '' |
272 | } |
273 | }, |
274 | 'todo' => {}, |
275 | 'totals' => { |
276 | 'bad' => 1, |
277 | 'bonus' => 0, |
278 | 'files' => 1, |
279 | 'good' => 0, |
280 | 'max' => 10, |
281 | 'ok' => 11, |
282 | 'skipped' => 0, |
283 | 'sub_skipped' => 0, |
284 | 'tests' => 1, |
285 | 'todo' => 0 |
286 | } |
287 | }, |
288 | 'head_end' => { |
289 | 'failed' => {}, |
290 | 'todo' => {}, |
291 | 'totals' => { |
292 | 'bad' => 0, |
293 | 'bonus' => 0, |
294 | 'files' => 1, |
295 | 'good' => 1, |
296 | 'max' => 4, |
297 | 'ok' => 4, |
298 | 'skipped' => 0, |
299 | 'sub_skipped' => 0, |
300 | 'tests' => 1, |
301 | 'todo' => 0 |
302 | } |
303 | }, |
304 | 'head_fail' => { |
305 | 'failed' => { |
306 | 't/sample-tests/head_fail' => { |
307 | 'canon' => 2, |
308 | 'estat' => '', |
309 | 'failed' => 1, |
310 | 'max' => 4, |
311 | 'name' => 't/sample-tests/head_fail', |
312 | 'wstat' => '' |
313 | } |
314 | }, |
315 | 'todo' => {}, |
316 | 'totals' => { |
317 | 'bad' => 1, |
318 | 'bonus' => 0, |
319 | 'files' => 1, |
320 | 'good' => 0, |
321 | 'max' => 4, |
322 | 'ok' => 3, |
323 | 'skipped' => 0, |
324 | 'sub_skipped' => 0, |
325 | 'tests' => 1, |
326 | 'todo' => 0 |
327 | } |
328 | }, |
329 | 'inc_taint' => { |
330 | 'failed' => { |
331 | 't/sample-tests/inc_taint' => { |
332 | 'canon' => 1, |
333 | 'estat' => 1, |
334 | 'failed' => 1, |
335 | 'max' => 1, |
336 | 'name' => 't/sample-tests/inc_taint', |
337 | 'wstat' => '256' |
338 | } |
339 | }, |
340 | 'todo' => {}, |
341 | 'totals' => { |
342 | 'bad' => 1, |
343 | 'bonus' => 0, |
344 | 'files' => 1, |
345 | 'good' => 0, |
346 | 'max' => 1, |
347 | 'ok' => 0, |
348 | 'skipped' => 0, |
349 | 'sub_skipped' => 0, |
350 | 'tests' => 1, |
351 | 'todo' => 0 |
352 | } |
353 | }, |
354 | 'junk_before_plan' => { |
355 | 'failed' => {}, |
356 | 'todo' => {}, |
357 | 'totals' => { |
358 | 'bad' => 0, |
359 | 'bonus' => 0, |
360 | 'files' => 1, |
361 | 'good' => 1, |
362 | 'max' => 1, |
363 | 'ok' => 1, |
364 | 'skipped' => 0, |
365 | 'sub_skipped' => 0, |
366 | 'tests' => 1, |
367 | 'todo' => 0 |
368 | } |
369 | }, |
370 | 'lone_not_bug' => { |
371 | 'failed' => {}, |
372 | 'todo' => {}, |
373 | 'totals' => { |
374 | 'bad' => 0, |
375 | 'bonus' => 0, |
376 | 'files' => 1, |
377 | 'good' => 1, |
378 | 'max' => 4, |
379 | 'ok' => 4, |
380 | 'skipped' => 0, |
381 | 'sub_skipped' => 0, |
382 | 'tests' => 1, |
383 | 'todo' => 0 |
384 | } |
385 | }, |
386 | 'no_nums' => { |
387 | 'failed' => { |
388 | 't/sample-tests/no_nums' => { |
389 | 'canon' => 3, |
390 | 'estat' => '', |
391 | 'failed' => 1, |
392 | 'max' => 5, |
393 | 'name' => 't/sample-tests/no_nums', |
394 | 'wstat' => '' |
395 | } |
396 | }, |
397 | 'todo' => {}, |
398 | 'totals' => { |
399 | 'bad' => 1, |
400 | 'bonus' => 0, |
401 | 'files' => 1, |
402 | 'good' => 0, |
403 | 'max' => 5, |
404 | 'ok' => 4, |
405 | 'skipped' => 0, |
406 | 'sub_skipped' => 0, |
407 | 'tests' => 1, |
408 | 'todo' => 0 |
409 | } |
410 | }, |
411 | 'no_output' => { |
412 | 'failed' => { |
413 | 't/sample-tests/no_output' => { |
414 | 'canon' => '??', |
415 | 'estat' => '', |
416 | 'failed' => '??', |
417 | 'max' => '??', |
418 | 'name' => 't/sample-tests/no_output', |
419 | 'wstat' => '' |
420 | } |
421 | }, |
422 | 'todo' => {}, |
423 | 'totals' => { |
424 | 'bad' => 1, |
425 | 'bonus' => 0, |
426 | 'files' => 1, |
427 | 'good' => 0, |
428 | 'max' => 0, |
429 | 'ok' => 0, |
430 | 'skipped' => 0, |
431 | 'sub_skipped' => 0, |
432 | 'tests' => 1, |
433 | 'todo' => 0 |
434 | } |
435 | }, |
436 | 'schwern' => { |
437 | 'failed' => {}, |
438 | 'todo' => {}, |
439 | 'totals' => { |
440 | 'bad' => 0, |
441 | 'bonus' => 0, |
442 | 'files' => 1, |
443 | 'good' => 1, |
444 | 'max' => 1, |
445 | 'ok' => 1, |
446 | 'skipped' => 0, |
447 | 'sub_skipped' => 0, |
448 | 'tests' => 1, |
449 | 'todo' => 0 |
450 | } |
451 | }, |
452 | 'sequence_misparse' => { |
453 | 'failed' => {}, |
454 | 'todo' => {}, |
455 | 'totals' => { |
456 | 'bad' => 0, |
457 | 'bonus' => 0, |
458 | 'files' => 1, |
459 | 'good' => 1, |
460 | 'max' => 5, |
461 | 'ok' => 5, |
462 | 'skipped' => 0, |
463 | 'sub_skipped' => 0, |
464 | 'tests' => 1, |
465 | 'todo' => 0 |
466 | } |
467 | }, |
468 | 'shbang_misparse' => { |
469 | 'failed' => {}, |
470 | 'todo' => {}, |
471 | 'totals' => { |
472 | 'bad' => 0, |
473 | 'bonus' => 0, |
474 | 'files' => 1, |
475 | 'good' => 1, |
476 | 'max' => 2, |
477 | 'ok' => 2, |
478 | 'skipped' => 0, |
479 | 'sub_skipped' => 0, |
480 | 'tests' => 1, |
481 | 'todo' => 0 |
482 | } |
483 | }, |
484 | 'simple' => { |
485 | 'failed' => {}, |
486 | 'todo' => {}, |
487 | 'totals' => { |
488 | 'bad' => 0, |
489 | 'bonus' => 0, |
490 | 'files' => 1, |
491 | 'good' => 1, |
492 | 'max' => 5, |
493 | 'ok' => 5, |
494 | 'skipped' => 0, |
495 | 'sub_skipped' => 0, |
496 | 'tests' => 1, |
497 | 'todo' => 0 |
498 | } |
499 | }, |
500 | 'simple_fail' => { |
501 | 'failed' => { |
502 | 't/sample-tests/simple_fail' => { |
503 | 'canon' => '2 5', |
504 | 'estat' => '', |
505 | 'failed' => 2, |
506 | 'max' => 5, |
507 | 'name' => 't/sample-tests/simple_fail', |
508 | 'wstat' => '' |
509 | } |
510 | }, |
511 | 'todo' => {}, |
512 | 'totals' => { |
513 | 'bad' => 1, |
514 | 'bonus' => 0, |
515 | 'files' => 1, |
516 | 'good' => 0, |
517 | 'max' => 5, |
518 | 'ok' => 3, |
519 | 'skipped' => 0, |
520 | 'sub_skipped' => 0, |
521 | 'tests' => 1, |
522 | 'todo' => 0 |
523 | } |
524 | }, |
525 | 'skip' => { |
526 | 'failed' => {}, |
527 | 'todo' => {}, |
528 | 'totals' => { |
529 | 'bad' => 0, |
530 | 'bonus' => 0, |
531 | 'files' => 1, |
532 | 'good' => 1, |
533 | 'max' => 5, |
534 | 'ok' => 5, |
535 | 'skipped' => 0, |
536 | 'sub_skipped' => 1, |
537 | 'tests' => 1, |
538 | 'todo' => 0 |
539 | } |
540 | }, |
541 | 'skip_nomsg' => { |
542 | 'failed' => {}, |
543 | 'todo' => {}, |
544 | 'totals' => { |
545 | 'bad' => 0, |
546 | 'bonus' => 0, |
547 | 'files' => 1, |
548 | 'good' => 1, |
549 | 'max' => 1, |
550 | 'ok' => 1, |
551 | 'skipped' => 0, |
552 | 'sub_skipped' => 1, |
553 | 'tests' => 1, |
554 | 'todo' => 0 |
555 | } |
556 | }, |
557 | 'skipall' => { |
558 | 'failed' => {}, |
559 | 'todo' => {}, |
560 | 'totals' => { |
561 | 'bad' => 0, |
562 | 'bonus' => 0, |
563 | 'files' => 1, |
564 | 'good' => 1, |
565 | 'max' => 0, |
566 | 'ok' => 0, |
567 | 'skipped' => 1, |
568 | 'sub_skipped' => 0, |
569 | 'tests' => 1, |
570 | 'todo' => 0 |
571 | } |
572 | }, |
573 | 'skipall_nomsg' => { |
574 | 'failed' => {}, |
575 | 'todo' => {}, |
576 | 'totals' => { |
577 | 'bad' => 0, |
578 | 'bonus' => 0, |
579 | 'files' => 1, |
580 | 'good' => 1, |
581 | 'max' => 0, |
582 | 'ok' => 0, |
583 | 'skipped' => 1, |
584 | 'sub_skipped' => 0, |
585 | 'tests' => 1, |
586 | 'todo' => 0 |
587 | } |
588 | }, |
589 | 'stdout_stderr' => { |
590 | 'failed' => {}, |
591 | 'todo' => {}, |
592 | 'totals' => { |
593 | 'bad' => 0, |
594 | 'bonus' => 0, |
595 | 'files' => 1, |
596 | 'good' => 1, |
597 | 'max' => 4, |
598 | 'ok' => 4, |
599 | 'skipped' => 0, |
600 | 'sub_skipped' => 0, |
601 | 'tests' => 1, |
602 | 'todo' => 0 |
603 | } |
604 | }, |
605 | 'switches' => { |
606 | 'failed' => { |
607 | 't/sample-tests/switches' => { |
608 | 'canon' => 1, |
609 | 'estat' => '', |
610 | 'failed' => 1, |
611 | 'max' => 1, |
612 | 'name' => 't/sample-tests/switches', |
613 | 'wstat' => '' |
614 | } |
615 | }, |
616 | 'todo' => {}, |
617 | 'totals' => { |
618 | 'bad' => 1, |
619 | 'bonus' => 0, |
620 | 'files' => 1, |
621 | 'good' => 0, |
622 | 'max' => 1, |
623 | 'ok' => 0, |
624 | 'skipped' => 0, |
625 | 'sub_skipped' => 0, |
626 | 'tests' => 1, |
627 | 'todo' => 0 |
628 | } |
629 | }, |
630 | 'taint' => { |
631 | 'failed' => {}, |
632 | 'todo' => {}, |
633 | 'totals' => { |
634 | 'bad' => 0, |
635 | 'bonus' => 0, |
636 | 'files' => 1, |
637 | 'good' => 1, |
638 | 'max' => 1, |
639 | 'ok' => 1, |
640 | 'skipped' => 0, |
641 | 'sub_skipped' => 0, |
642 | 'tests' => 1, |
643 | 'todo' => 0 |
644 | } |
645 | }, |
646 | 'taint_warn' => { |
647 | 'failed' => {}, |
648 | 'todo' => {}, |
649 | 'totals' => { |
650 | 'bad' => 0, |
651 | 'bonus' => 0, |
652 | 'files' => 1, |
653 | 'good' => 1, |
654 | 'max' => 1, |
655 | 'ok' => 1, |
656 | 'skipped' => 0, |
657 | 'sub_skipped' => 0, |
658 | 'tests' => 1, |
659 | 'todo' => 0 |
660 | }, |
661 | 'require' => 5.008001, |
662 | }, |
663 | 'todo_inline' => { |
664 | 'failed' => {}, |
665 | 'todo' => { |
666 | 't/sample-tests/todo_inline' => { |
667 | 'canon' => 2, |
668 | 'estat' => '', |
669 | 'failed' => 1, |
670 | 'max' => 2, |
671 | 'name' => 't/sample-tests/todo_inline', |
672 | 'wstat' => '' |
673 | } |
674 | }, |
675 | 'totals' => { |
676 | 'bad' => 0, |
677 | 'bonus' => 1, |
678 | 'files' => 1, |
679 | 'good' => 1, |
680 | 'max' => 3, |
681 | 'ok' => 3, |
682 | 'skipped' => 0, |
683 | 'sub_skipped' => 0, |
684 | 'tests' => 1, |
685 | 'todo' => 2 |
686 | } |
687 | }, |
688 | 'todo_misparse' => { |
689 | 'failed' => { |
690 | 't/sample-tests/todo_misparse' => { |
691 | 'canon' => 1, |
692 | 'estat' => '', |
693 | 'failed' => 1, |
694 | 'max' => 1, |
695 | 'name' => 't/sample-tests/todo_misparse', |
696 | 'wstat' => '' |
697 | } |
698 | }, |
699 | 'todo' => {}, |
700 | 'totals' => { |
701 | 'bad' => 1, |
702 | 'bonus' => 0, |
703 | 'files' => 1, |
704 | 'good' => 0, |
705 | 'max' => 1, |
706 | 'ok' => 0, |
707 | 'skipped' => 0, |
708 | 'sub_skipped' => 0, |
709 | 'tests' => 1, |
710 | 'todo' => 0 |
711 | } |
712 | }, |
713 | 'too_many' => { |
714 | 'failed' => { |
715 | 't/sample-tests/too_many' => { |
716 | 'canon' => '4-7', |
717 | 'estat' => 4, |
718 | 'failed' => 4, |
719 | 'max' => 3, |
720 | 'name' => 't/sample-tests/too_many', |
721 | 'wstat' => '1024' |
722 | } |
723 | }, |
724 | 'todo' => {}, |
725 | 'totals' => { |
726 | 'bad' => 1, |
727 | 'bonus' => 0, |
728 | 'files' => 1, |
729 | 'good' => 0, |
730 | 'max' => 3, |
731 | 'ok' => 7, |
732 | 'skipped' => 0, |
733 | 'sub_skipped' => 0, |
734 | 'tests' => 1, |
735 | 'todo' => 0 |
736 | } |
737 | }, |
738 | 'vms_nit' => { |
739 | 'failed' => { |
740 | 't/sample-tests/vms_nit' => { |
741 | 'canon' => 1, |
742 | 'estat' => '', |
743 | 'failed' => 1, |
744 | 'max' => 2, |
745 | 'name' => 't/sample-tests/vms_nit', |
746 | 'wstat' => '' |
747 | } |
748 | }, |
749 | 'todo' => {}, |
750 | 'totals' => { |
751 | 'bad' => 1, |
752 | 'bonus' => 0, |
753 | 'files' => 1, |
754 | 'good' => 0, |
755 | 'max' => 2, |
756 | 'ok' => 1, |
757 | 'skipped' => 0, |
758 | 'sub_skipped' => 0, |
759 | 'tests' => 1, |
760 | 'todo' => 0 |
761 | } |
762 | } |
763 | }; |
764 | |
765 | my $num_tests = ( keys %$results ) * $PER_LOOP; |
766 | |
767 | plan tests => $num_tests; |
768 | |
769 | sub local_name { |
770 | my $name = shift; |
771 | return File::Spec->catfile( split /\//, $name ); |
772 | } |
773 | |
774 | sub local_result { |
775 | my $hash = shift; |
776 | my $new = {}; |
777 | |
778 | while ( my ( $file, $want ) = each %$hash ) { |
779 | if ( exists $want->{name} ) { |
780 | $want->{name} = local_name( $want->{name} ); |
781 | } |
782 | $new->{ local_name($file) } = $want; |
783 | } |
784 | return $new; |
785 | } |
786 | |
787 | sub vague_status { |
788 | my $hash = shift; |
789 | return $hash unless $^O eq 'VMS'; |
790 | |
791 | while ( my ( $file, $want ) = each %$hash ) { |
792 | for ( qw( estat wstat ) ) { |
793 | if ( exists $want->{$_} ) { |
794 | $want->{$_} = $want->{$_} ? 1 : 0; |
795 | } |
796 | } |
797 | } |
798 | return $hash |
799 | } |
800 | |
801 | { |
802 | local $^W = 0; |
803 | |
804 | # Silence harness output |
805 | *TAP::Formatter::Console::_output = sub { |
806 | |
807 | # do nothing |
808 | }; |
809 | } |
810 | |
811 | for my $test_key ( sort keys %$results ) { |
812 | my $result = $results->{$test_key}; |
813 | SKIP: { |
814 | if ( $result->{require} && $] < $result->{require} ) { |
815 | skip "Test requires Perl $result->{require}, we have $]", 4; |
816 | } |
817 | my @test_names = split( /,/, $test_key ); |
818 | my @test_files |
819 | = map { File::Spec->catfile( $TEST_DIR, $_ ) } @test_names; |
820 | |
821 | # For now we supress STDERR because it crufts up /our/ test |
822 | # results. Should probably capture and analyse it. |
823 | local ( *OLDERR, *OLDOUT ); |
824 | open OLDERR, '>&STDERR' or die $!; |
825 | open OLDOUT, '>&STDOUT' or die $!; |
826 | my $devnull = File::Spec->devnull; |
827 | open STDERR, ">$devnull" or die $!; |
828 | open STDOUT, ">$devnull" or die $!; |
829 | |
830 | my ( $tot, $fail, $todo, $harness, $aggregate ) |
831 | = execute_tests( tests => \@test_files ); |
832 | |
833 | open STDERR, '>&OLDERR' or die $!; |
834 | open STDOUT, '>&OLDOUT' or die $!; |
835 | |
836 | my $bench = delete $tot->{bench}; |
837 | isa_ok $bench, 'Benchmark'; |
838 | |
839 | # Localise filenames in failed, todo |
840 | my $lfailed = vague_status( local_result( $result->{failed} ) ); |
841 | my $ltodo = vague_status( local_result( $result->{todo} ) ); |
842 | |
843 | # use Data::Dumper; |
844 | # diag Dumper( [ $lfailed, $ltodo ] ); |
845 | |
846 | is_deeply $tot, $result->{totals}, "totals match for $test_key"; |
847 | is_deeply vague_status($fail), $lfailed, |
848 | "failure summary matches for $test_key"; |
849 | is_deeply vague_status($todo), $ltodo, |
850 | "todo summary matches for $test_key"; |
851 | } |
852 | } |
853 | } |