#!/usr/bin/python3
"""Regression tests for Pacemaker's scheduler."""

# pylint doesn't like the module name "cts-scheduler" which is an invalid complaint for this file
# but probably something we want to continue warning about elsewhere
# pylint: disable=invalid-name
# pacemaker imports need to come after we modify sys.path, which pylint will complain about.
# pylint: disable=wrong-import-position

__copyright__ = "Copyright 2004-2025 the Pacemaker project contributors"
__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"

import io
import os
import re
import sys
import stat
import shlex
import shutil
import argparse
import subprocess
import platform
import tempfile

# These imports allow running from a source checkout after running `make`.
# Note that while this doesn't necessarily mean it will successfully run tests,
# but being able to see --help output can be useful.
if os.path.exists("/builddir/build/BUILD/pacemaker-9a5e54bae/python"):
    sys.path.insert(0, "/builddir/build/BUILD/pacemaker-9a5e54bae/python")

# pylint: disable=comparison-of-constants,comparison-with-itself,condition-evals-to-constant
if os.path.exists("/builddir/build/BUILD/pacemaker-9a5e54bae/python") and "/builddir/build/BUILD/pacemaker-9a5e54bae" != "/builddir/build/BUILD/pacemaker-9a5e54bae":
    sys.path.insert(0, "/builddir/build/BUILD/pacemaker-9a5e54bae/python")

from pacemaker.buildoptions import BuildOptions
from pacemaker.exitstatus import ExitStatus


class SchedulerTest:
    """A single scheduler test."""

    def __init__(self, name, desc, args=None):
        """
        Create a new SchedulerTest instance.

        Arguments:
        name -- A unique name for this test.
        desc -- A meaningful description for the test.
        args -- Additional arguments to pass when running this test
        """
        self.name = name
        self.desc = desc

        if args is None:
            self.args = []
        else:
            self.args = args


class SchedulerTestGroup:
    """Collection of scheduler regression tests."""

    def __init__(self, tests):
        """
        Create a new SchedulerTestGroup instance.

        Arguments:
        tests -- A list of SchedulerTest instances to be executed as part of
                 this group.
        """
        self.tests = tests


# Each entry in TESTS is a group of tests, where each test consists of a
# test base name, test description, and additional test arguments.
# Test groups will be separated by newlines in output.
TESTS = [
    SchedulerTestGroup([
        SchedulerTest("simple1", "Offline"),
        SchedulerTest("simple2", "Start"),
        SchedulerTest("simple3", "Start 2"),
        SchedulerTest("simple4", "Start Failed"),
        SchedulerTest("simple6", "Stop Start"),
        SchedulerTest("simple7", "Shutdown"),
        SchedulerTest("simple11", "Priority (ne)"),
        SchedulerTest("simple12", "Priority (eq)"),
        SchedulerTest("simple8", "Stickiness"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("group1", "Group"),
        SchedulerTest("group2", "Group + Native"),
        SchedulerTest("group3", "Group + Group"),
        SchedulerTest("group4", "Group + Native (nothing)"),
        SchedulerTest("group5", "Group + Native (move)"),
        SchedulerTest("group6", "Group + Group (move)"),
        SchedulerTest("group7", "Group colocation"),
        SchedulerTest("group13", "Group colocation (cant run)"),
        SchedulerTest("group8", "Group anti-colocation"),
        SchedulerTest("group9", "Group recovery"),
        SchedulerTest("group10", "Group partial recovery"),
        SchedulerTest("group11", "Group target_role"),
        SchedulerTest("group14", "Group stop (graph terminated)"),
        SchedulerTest("group15", "Negative group colocation"),
        SchedulerTest("bug-1573", "Partial stop of a group with two children"),
        SchedulerTest("bug-1718", "Mandatory group ordering - Stop group_FUN"),
        SchedulerTest("failed-sticky-group", "Move group on last member failure despite infinite stickiness"),
        SchedulerTest("failed-sticky-anticolocated-group",
                      "Move group on last member failure despite infinite stickiness and optional anti-colocation"),
        SchedulerTest("bug-lf-2619", "Move group on clone failure"),
        SchedulerTest("group-fail", "Ensure stop order is preserved for partially active groups"),
        SchedulerTest("group-unmanaged", "No need to restart r115 because r114 is unmanaged"),
        SchedulerTest("group-unmanaged-stopped", "Make sure r115 is stopped when r114 fails"),
        SchedulerTest("partial-unmanaged-group", "New member in partially unmanaged group"),
        SchedulerTest("group-dependents", "Account for the location preferences of things colocated with a group"),
        SchedulerTest("group-stop-ordering", "Ensure blocked group member stop does not force other member stops"),
        SchedulerTest("colocate-unmanaged-group", "Respect mandatory colocations even if earlier group member is unmanaged"),
        SchedulerTest("coloc-with-inner-group-member", "Consider explicit colocations with inner group members"),
        SchedulerTest("banned-group-inner-constraints",
                      "Group banned from current node, inner member constrained"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("rsc_dep1", "Must not"),
        SchedulerTest("rsc_dep3", "Must"),
        SchedulerTest("rsc_dep5", "Must not 3"),
        SchedulerTest("rsc_dep7", "Must 3"),
        SchedulerTest("rsc_dep10", "Must (but cant)"),
        SchedulerTest("rsc_dep2", "Must (running)"),
        SchedulerTest("rsc_dep8", "Must (running : alt)"),
        SchedulerTest("rsc_dep4", "Must (running + move)"),
        SchedulerTest("asymmetric", "Asymmetric - require explicit location constraints"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("orphan-0", "Orphan ignore"),
        SchedulerTest("orphan-1", "Orphan stop"),
        SchedulerTest("orphan-2", "Orphan stop, remove failcount"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("params-0", "Params: No change"),
        SchedulerTest("params-1", "Params: Changed"),
        SchedulerTest("params-2", "Params: Resource definition"),
        SchedulerTest("params-3", "Params: Restart instead of reload if start pending"),
        SchedulerTest("params-4", "Params: Reload"),
        SchedulerTest("params-5", "Params: Restart based on probe digest"),
        SchedulerTest("novell-251689", "Resource definition change + target_role=stopped"),
        SchedulerTest("bug-lf-2106", "Restart all anonymous clone instances after config change"),
        SchedulerTest("params-6", "Params: Detect reload in previously migrated resource"),
        SchedulerTest("nvpair-id-ref", "Support id-ref in nvpair with optional name"),
        SchedulerTest("not-reschedule-unneeded-monitor",
                      "Do not reschedule unneeded monitors while resource definitions have changed"),
        SchedulerTest("reload-becomes-restart", "Cancel reload if restart becomes required"),
        SchedulerTest("restart-with-extra-op-params", "Restart if with extra operation parameters upon changes of any"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("target-0", "Target Role : baseline"),
        SchedulerTest("target-1", "Target Role : promoted"),
        SchedulerTest("target-2", "Target Role : invalid"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("base-score", "Set a node's default score for all nodes"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("date-1", "Dates", ["-t", "2005-020"]),
        SchedulerTest("date-2", "Date Spec - Pass", ["-t", "2005-020T12:30"]),
        SchedulerTest("date-3", "Date Spec - Fail", ["-t", "2005-020T11:30"]),
        SchedulerTest("origin", "Timing of recurring operations", ["-t", "2014-05-07 00:28:00"]),
        SchedulerTest("probe-0", "Probe (anon clone)"),
        SchedulerTest("probe-1", "Pending Probe"),
        SchedulerTest("probe-2", "Correctly re-probe cloned groups"),
        SchedulerTest("probe-3", "Probe (pending node)"),
        SchedulerTest("probe-4", "Probe (pending node + stopped resource)"),
        SchedulerTest("probe-pending-node", "Probe (pending node + unmanaged resource)"),
        SchedulerTest("failed-probe-primitive", "Maskable vs. unmaskable probe failures on primitive resources"),
        SchedulerTest("failed-probe-clone", "Maskable vs. unmaskable probe failures on cloned resources"),
        SchedulerTest("expired-failed-probe-primitive", "Maskable, expired probe failure on primitive resources"),
        SchedulerTest("standby", "Standby"),
        SchedulerTest("comments", "Comments"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("one-or-more-0", "Everything starts"),
        SchedulerTest("one-or-more-1", "Nothing starts because of A"),
        SchedulerTest("one-or-more-2", "D can start because of C"),
        SchedulerTest("one-or-more-3", "D cannot start because of B and C"),
        SchedulerTest("one-or-more-4", "D cannot start because of target-role"),
        SchedulerTest("one-or-more-5", "Start A and F even though C and D are stopped"),
        SchedulerTest("one-or-more-6", "Leave A running even though B is stopped"),
        SchedulerTest("one-or-more-7", "Leave A running even though C is stopped"),
        SchedulerTest("bug-5140-require-all-false", "Allow basegrp:0 to stop"),
        SchedulerTest("clone-require-all-1", "clone B starts node 3 and 4"),
        SchedulerTest("clone-require-all-2", "clone B remains stopped everywhere"),
        SchedulerTest("clone-require-all-3", "clone B stops everywhere because A stops everywhere"),
        SchedulerTest("clone-require-all-4", "clone B remains on node 3 and 4 with only one instance of A remaining"),
        SchedulerTest("clone-require-all-5", "clone B starts on node 1 3 and 4"),
        SchedulerTest("clone-require-all-6", "clone B remains active after shutting down instances of A"),
        SchedulerTest("clone-require-all-7",
                      "clone A and B both start at the same time. all instances of A start before B"),
        SchedulerTest("clone-require-all-no-interleave-1", "C starts everywhere after A and B"),
        SchedulerTest("clone-require-all-no-interleave-2",
                      "C starts on nodes 1, 2, and 4 with only one active instance of B"),
        SchedulerTest("clone-require-all-no-interleave-3",
                      "C remains active when instance of B is stopped on one node and started on another"),
        SchedulerTest("one-or-more-unrunnable-instances", "Avoid dependencies on instances that won't ever be started"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("location-date-rules-1", "Use location constraints with ineffective date-based rules"),
        SchedulerTest("location-date-rules-2", "Use location constraints with effective date-based rules"),
        SchedulerTest("nvpair-date-rules-1", "Use nvpair blocks with a variety of date-based rules"),
        SchedulerTest("value-source", "Use location constraints with node attribute expressions using value-source"),
        SchedulerTest("rule-dbl-as-auto-number-match",
                      "Floating-point rule values default to number comparison: match"),
        SchedulerTest("rule-dbl-as-auto-number-no-match",
                      "Floating-point rule values default to number comparison: no match"),
        SchedulerTest("rule-dbl-as-integer-match",
                      "Floating-point rule values set to integer comparison: match"),
        SchedulerTest("rule-dbl-as-integer-no-match",
                      "Floating-point rule values set to integer comparison: no match"),
        SchedulerTest("rule-dbl-as-number-match",
                      "Floating-point rule values set to number comparison: match"),
        SchedulerTest("rule-dbl-as-number-no-match",
                      "Floating-point rule values set to number comparison: no match"),
        SchedulerTest("rule-dbl-parse-fail-default-str-match",
                      "Floating-point rule values fail to parse, default to string comparison: match"),
        SchedulerTest("rule-dbl-parse-fail-default-str-no-match",
                      "Floating-point rule values fail to parse, default to string comparison: no match"),
        SchedulerTest("rule-int-as-auto-integer-match",
                      "Integer rule values default to integer comparison: match"),
        SchedulerTest("rule-int-as-auto-integer-no-match",
                      "Integer rule values default to integer comparison: no match"),
        SchedulerTest("rule-int-as-integer-match",
                      "Integer rule values set to integer comparison: match"),
        SchedulerTest("rule-int-as-integer-no-match",
                      "Integer rule values set to integer comparison: no match"),
        SchedulerTest("rule-int-as-number-match",
                      "Integer rule values set to number comparison: match"),
        SchedulerTest("rule-int-as-number-no-match",
                      "Integer rule values set to number comparison: no match"),
        SchedulerTest("rule-int-parse-fail-default-str-match",
                      "Integer rule values fail to parse, default to string comparison: match"),
        SchedulerTest("rule-int-parse-fail-default-str-no-match",
                      "Integer rule values fail to parse, default to string comparison: no match"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("order1", "Order start 1"),
        SchedulerTest("order2", "Order start 2"),
        SchedulerTest("order3", "Order stop"),
        SchedulerTest("order4", "Order (multiple)"),
        SchedulerTest("order5", "Order (move)"),
        SchedulerTest("order6", "Order (move w/ restart)"),
        SchedulerTest("order7", "Order (mandatory)"),
        SchedulerTest("order-optional", "Order (score=0)"),
        SchedulerTest("order-required", "Order (score=INFINITY)"),
        SchedulerTest("bug-lf-2171", "Prevent group start when clone is stopped"),
        SchedulerTest("order-clone", "Clone ordering should be able to prevent startup of dependent clones"),
        SchedulerTest("order-sets", "Ordering for resource sets"),
        SchedulerTest("order-serialize", "Serialize resources without inhibiting migration"),
        SchedulerTest("order-serialize-set", "Serialize a set of resources without inhibiting migration"),
        SchedulerTest("clone-order-primitive", "Order clone start after a primitive"),
        SchedulerTest("clone-order-16instances", "Verify ordering of 16 cloned resources"),
        SchedulerTest("order-optional-keyword", "Order (optional keyword)"),
        SchedulerTest("order-mandatory", "Order (mandatory keyword)"),
        SchedulerTest("bug-lf-2493", "Don't imply colocation requirements when applying ordering constraints with clones"),
        SchedulerTest("ordered-set-basic-startup", "Constraint set with default order settings"),
        SchedulerTest("ordered-set-natural", "Allow natural set ordering"),
        SchedulerTest("order-wrong-kind", "Order (error)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("coloc-loop", "Colocation - loop"),
        SchedulerTest("coloc-many-one", "Colocation - many-to-one"),
        SchedulerTest("coloc-list", "Colocation - many-to-one with list"),
        SchedulerTest("coloc-group", "Colocation - groups"),
        SchedulerTest("coloc-unpromoted-anti", "Anti-colocation with unpromoted shouldn't prevent promoted colocation"),
        SchedulerTest("coloc-attr", "Colocation based on node attributes"),
        SchedulerTest("coloc-negative-group", "Negative colocation with a group"),
        SchedulerTest("coloc-intra-set", "Intra-set colocation"),
        SchedulerTest("bug-lf-2435", "Colocation sets with a negative score"),
        SchedulerTest("coloc-clone-stays-active",
                      "Ensure clones don't get stopped/demoted because a dependent must stop"),
        SchedulerTest("coloc_fp_logic", "Verify floating point calculations in colocation are working"),
        SchedulerTest("colo_promoted_w_native",
                      "cl#5070 - Verify promotion order is affected when colocating promoted with primitive"),
        SchedulerTest("colo_unpromoted_w_native",
                      "cl#5070 - Verify promotion order is affected when colocating unpromoted with primitive"),
        SchedulerTest("anti-colocation-order",
                      "cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node"),
        SchedulerTest("anti-colocation-promoted", "Organize order of actions for promoted resources in anti-colocations"),
        SchedulerTest("anti-colocation-unpromoted", "Organize order of actions for unpromoted resources in anti-colocations"),
        SchedulerTest("group-anticolocation", "Group with failed last member anti-colocated with another group"),
        SchedulerTest("group-anticolocation-2",
                      "Group with failed last member anti-colocated with another sticky group"),
        SchedulerTest("group-anticolocation-3",
                      "Group with failed last member mandatorily anti-colocated with another group"),
        SchedulerTest("group-anticolocation-4",
                      "Group with failed last member anti-colocated without influence with another group"),
        SchedulerTest("group-anticolocation-5",
                      "Group with failed last member anti-colocated with another group (third node allowed)"),
        SchedulerTest("group-colocation-failure",
                      "Group with sole member failed, colocated with another group"),
        SchedulerTest("enforce-colo1", "Always enforce B with A INFINITY"),
        SchedulerTest("complex_enforce_colo", "Always enforce B with A INFINITY. (make sure heat-engine stops)"),
        SchedulerTest("coloc-dependee-should-stay", "Stickiness outweighs group colocation"),
        SchedulerTest("coloc-dependee-should-move", "Group colocation outweighs stickiness"),
        SchedulerTest("colocation-influence", "Respect colocation influence"),
        SchedulerTest("colocation-priority-group", "Apply group colocations in order of primary priority"),
        SchedulerTest("colocation-vs-stickiness", "Group stickiness outweighs anti-colocation score"),
        SchedulerTest("promoted-with-blocked", "Promoted role colocated with a resource with blocked start"),
        SchedulerTest("primitive-with-group-with-clone",
                      "Consider group dependent when colocating with clone"),
        SchedulerTest("primitive-with-group-with-promoted",
                      "Consider group dependent when colocating with promoted role"),
        SchedulerTest("primitive-with-unrunnable-group",
                      "Block primitive colocated with group that can't start"),
        SchedulerTest("coloc-cloned-group-promoted-dependent1",
                      "Cloned group promoted role with primitive (mandatory)"),
        SchedulerTest("coloc-cloned-group-promoted-dependent2",
                      "Cloned group promoted role with primitive (optional)"),
        SchedulerTest("coloc-optional-promoted-dependent-moves-1",
                      "Colocation score less than promotion score difference: move"),
        SchedulerTest("coloc-optional-promoted-dependent-moves-2",
                      "Colocation score greater than promotion score difference: move"),
        SchedulerTest("coloc-optional-promoted-dependent-stays-1",
                      "Colocation score greater than promotion score difference: stay"),
        SchedulerTest("coloc-optional-promoted-dependent-stays-2",
                      "Colocation score less than promotion score difference: stay"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("rsc-sets-seq-true", "Resource Sets - sequential=false"),
        SchedulerTest("rsc-sets-seq-false", "Resource Sets - sequential=true"),
        SchedulerTest("rsc-sets-clone", "Resource Sets - Clone"),
        SchedulerTest("rsc-sets-promoted", "Resource Sets - Promoted"),
        SchedulerTest("rsc-sets-clone-1", "Resource Sets - Clone (lf#2404)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("attrs1", "string: eq (and)"),
        SchedulerTest("attrs2", "string: lt / gt (and)"),
        SchedulerTest("attrs3", "string: ne (or)"),
        SchedulerTest("attrs4", "string: exists"),
        SchedulerTest("attrs5", "string: not_exists"),
        SchedulerTest("attrs6", "is_dc: true"),
        SchedulerTest("attrs7", "is_dc: false"),
        SchedulerTest("attrs8", "score_attribute"),
        SchedulerTest("per-node-attrs", "Per node resource parameters"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("mon-rsc-1", "Schedule Monitor - start"),
        SchedulerTest("mon-rsc-2", "Schedule Monitor - move"),
        SchedulerTest("mon-rsc-3", "Schedule Monitor - pending start"),
        SchedulerTest("mon-rsc-4", "Schedule Monitor - move/pending start"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("rec-rsc-0", "Resource Recover - no start"),
        SchedulerTest("rec-rsc-1", "Resource Recover - start"),
        SchedulerTest("rec-rsc-2", "Resource Recover - monitor"),
        SchedulerTest("rec-rsc-3", "Resource Recover - stop - ignore"),
        SchedulerTest("rec-rsc-4", "Resource Recover - stop - block"),
        SchedulerTest("rec-rsc-5", "Resource Recover - stop - fence"),
        SchedulerTest("rec-rsc-6", "Resource Recover - multiple - restart"),
        SchedulerTest("rec-rsc-7", "Resource Recover - multiple - stop"),
        SchedulerTest("rec-rsc-8", "Resource Recover - multiple - block"),
        SchedulerTest("rec-rsc-9", "Resource Recover - group/group"),
        SchedulerTest("stop-unexpected", "Recover multiply active group with stop_unexpected"),
        SchedulerTest("stop-unexpected-2", "Resource multiply active primitve with stop_unexpected"),
        SchedulerTest("monitor-recovery", "on-fail=block + resource recovery detected by recurring monitor"),
        SchedulerTest("stop-failure-no-quorum", "Stop failure without quorum"),
        SchedulerTest("stop-failure-no-fencing", "Stop failure without fencing available"),
        SchedulerTest("stop-failure-with-fencing", "Stop failure with fencing available"),
        SchedulerTest("multiple-active-block-group", "Support of multiple-active=block for resource groups"),
        SchedulerTest("multiple-monitor-one-failed",
                      "Consider resource failed if any of the configured monitor operations failed"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("quorum-1", "No quorum - ignore"),
        SchedulerTest("quorum-2", "No quorum - freeze"),
        SchedulerTest("quorum-3", "No quorum - stop"),
        SchedulerTest("quorum-4", "No quorum - start anyway"),
        SchedulerTest("quorum-5", "No quorum - start anyway (group)"),
        SchedulerTest("quorum-6", "No quorum - start anyway (clone)"),
        SchedulerTest("bug-cl-5212", "No promotion with no-quorum-policy=freeze"),
        SchedulerTest("suicide-needed-inquorate", "no-quorum-policy=suicide: suicide necessary"),
        SchedulerTest("suicide-not-needed-initial-quorum",
                      "no-quorum-policy=suicide: suicide not necessary at initial quorum"),
        SchedulerTest("suicide-not-needed-never-quorate",
                      "no-quorum-policy=suicide: suicide not necessary if never quorate"),
        SchedulerTest("suicide-not-needed-quorate", "no-quorum-policy=suicide: suicide necessary if quorate"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("rec-node-1", "Node Recover - Startup   - no fence"),
        SchedulerTest("rec-node-2", "Node Recover - Startup   - fence"),
        SchedulerTest("rec-node-3", "Node Recover - HA down   - no fence"),
        SchedulerTest("rec-node-4", "Node Recover - HA down   - fence"),
        SchedulerTest("rec-node-5", "Node Recover - CRM down  - no fence"),
        SchedulerTest("rec-node-6", "Node Recover - CRM down  - fence"),
        SchedulerTest("rec-node-7", "Node Recover - no quorum - ignore"),
        SchedulerTest("rec-node-8", "Node Recover - no quorum - freeze"),
        SchedulerTest("rec-node-9", "Node Recover - no quorum - stop"),
        SchedulerTest("rec-node-10", "Node Recover - no quorum - stop w/fence"),
        SchedulerTest("rec-node-11", "Node Recover - CRM down w/ group - fence"),
        SchedulerTest("rec-node-12", "Node Recover - nothing active - fence"),
        SchedulerTest("rec-node-13", "Node Recover - failed resource + shutdown - fence"),
        SchedulerTest("rec-node-15", "Node Recover - unknown lrm section"),
        SchedulerTest("rec-node-14", "Serialize all stonith's"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("multi1", "Multiple Active (stop/start)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("migrate-begin", "Normal migration"),
        SchedulerTest("migrate-success", "Completed migration"),
        SchedulerTest("migrate-partial-1", "Completed migration, missing stop on source"),
        SchedulerTest("migrate-partial-2", "Successful migrate_to only"),
        SchedulerTest("migrate-partial-3", "Successful migrate_to only, target down"),
        SchedulerTest("migrate-partial-4", "Migrate from the correct host after migrate_to+migrate_from"),
        SchedulerTest("bug-5186-partial-migrate", "Handle partial migration when src node loses membership"),
        SchedulerTest("migrate-fail-2", "Failed migrate_from"),
        SchedulerTest("migrate-fail-3", "Failed migrate_from + stop on source"),
        SchedulerTest("migrate-fail-4",
                      "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target"),
        SchedulerTest("migrate-fail-5", "Failed migrate_from + stop on source and target"),
        SchedulerTest("migrate-fail-6", "Failed migrate_to"),
        SchedulerTest("migrate-fail-7", "Failed migrate_to + stop on source"),
        SchedulerTest("migrate-fail-8",
                      "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target"),
        SchedulerTest("migrate-fail-9", "Failed migrate_to + stop on source and target"),
        SchedulerTest("migration-ping-pong", "Old migrate_to failure + successful migrate_from on same node"),
        SchedulerTest("migrate-stop", "Migration in a stopping stack"),
        SchedulerTest("migrate-start", "Migration in a starting stack"),
        SchedulerTest("migrate-stop_start", "Migration in a restarting stack"),
        SchedulerTest("migrate-stop-complex", "Migration in a complex stopping stack"),
        SchedulerTest("migrate-start-complex", "Migration in a complex starting stack"),
        SchedulerTest("migrate-stop-start-complex", "Migration in a complex moving stack"),
        SchedulerTest("migrate-shutdown", "Order the post-migration 'stop' before node shutdown"),
        SchedulerTest("migrate-1", "Migrate (migrate)"),
        SchedulerTest("migrate-2", "Migrate (stable)"),
        SchedulerTest("migrate-3", "Migrate (failed migrate_to)"),
        SchedulerTest("migrate-4", "Migrate (failed migrate_from)"),
        SchedulerTest("novell-252693", "Migration in a stopping stack"),
        SchedulerTest("novell-252693-2", "Migration in a starting stack"),
        SchedulerTest("novell-252693-3", "Non-Migration in a starting and stopping stack"),
        SchedulerTest("bug-1820", "Migration in a group"),
        SchedulerTest("bug-1820-1", "Non-migration in a group"),
        SchedulerTest("migrate-5", "Primitive migration with a clone"),
        SchedulerTest("migrate-fencing", "Migration after Fencing"),
        SchedulerTest("migrate-both-vms", "Migrate two VMs that have no colocation"),
        SchedulerTest("migration-behind-migrating-remote", "Migrate resource behind migrating remote connection"),
        SchedulerTest("1-a-then-bm-move-b", "Advanced migrate logic. A then B. migrate B"),
        SchedulerTest("2-am-then-b-move-a", "Advanced migrate logic, A then B, migrate A without stopping B"),
        SchedulerTest("3-am-then-bm-both-migrate", "Advanced migrate logic. A then B. migrate both"),
        SchedulerTest("4-am-then-bm-b-not-migratable", "Advanced migrate logic, A then B, B not migratable"),
        SchedulerTest("5-am-then-bm-a-not-migratable", "Advanced migrate logic. A then B. move both, a not migratable"),
        SchedulerTest("6-migrate-group", "Advanced migrate logic, migrate a group"),
        SchedulerTest("7-migrate-group-one-unmigratable",
                      "Advanced migrate logic, migrate group mixed with allow-migrate true/false"),
        SchedulerTest("8-am-then-bm-a-migrating-b-stopping",
                      "Advanced migrate logic, A then B, A migrating, B stopping"),
        SchedulerTest("9-am-then-bm-b-migrating-a-stopping",
                      "Advanced migrate logic, A then B, B migrate, A stopping"),
        SchedulerTest("10-a-then-bm-b-move-a-clone",
                      "Advanced migrate logic, A clone then B, migrate B while stopping A"),
        SchedulerTest("11-a-then-bm-b-move-a-clone-starting",
                      "Advanced migrate logic, A clone then B, B moving while A is start/stopping"),
        SchedulerTest("a-promote-then-b-migrate", "A promote then B start. migrate B"),
        SchedulerTest("a-demote-then-b-migrate", "A demote then B stop. migrate B"),
        SchedulerTest("probe-target-of-failed-migrate_to-1", "Failed migrate_to, target rejoins"),
        SchedulerTest("probe-target-of-failed-migrate_to-2", "Failed migrate_to, target rejoined and probed"),
        SchedulerTest("partial-live-migration-multiple-active", "Prevent running on multiple nodes due to partial live migration"),
        SchedulerTest("migration-intermediary-cleaned",
                      "Probe live-migration intermediary with no history"),
        SchedulerTest("bug-lf-2422", "Dependency on partially active group - stop ocfs:*"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("clone-anon-probe-1", "Probe the correct (anonymous) clone instance for each node"),
        SchedulerTest("clone-anon-probe-2", "Avoid needless re-probing of anonymous clones"),
        SchedulerTest("clone-anon-failcount", "Merge failcounts for anonymous clones"),
        SchedulerTest("force-anon-clone-max", "Update clone-max properly when forcing a clone to be anonymous"),
        SchedulerTest("anon-instance-pending", "Assign anonymous clone instance numbers properly when action pending"),
        SchedulerTest("inc0", "Incarnation start"),
        SchedulerTest("inc1", "Incarnation start order"),
        SchedulerTest("inc2", "Incarnation silent restart, stop, move"),
        SchedulerTest("inc3", "Inter-incarnation ordering, silent restart, stop, move"),
        SchedulerTest("inc4", "Inter-incarnation ordering, silent restart, stop, move (ordered)"),
        SchedulerTest("inc5", "Inter-incarnation ordering, silent restart, stop, move (restart 1)"),
        SchedulerTest("inc6", "Inter-incarnation ordering, silent restart, stop, move (restart 2)"),
        SchedulerTest("inc7", "Clone colocation"),
        SchedulerTest("inc8", "Clone anti-colocation"),
        SchedulerTest("inc9", "Non-unique clone"),
        SchedulerTest("inc10", "Non-unique clone (stop)"),
        SchedulerTest("inc11", "Primitive colocation with clones"),
        SchedulerTest("inc12", "Clone shutdown"),
        SchedulerTest("cloned-group", "Make sure only the correct number of cloned groups are started"),
        SchedulerTest("cloned-group-stop", "Ensure stopping qpidd also stops glance and cinder"),
        SchedulerTest("clone-no-shuffle", "Don't prioritize allocation of instances that must be moved"),
        SchedulerTest("clone-recover-no-shuffle-1",
                      "Don't shuffle instances when starting a new primitive instance"),
        SchedulerTest("clone-recover-no-shuffle-2",
                      "Don't shuffle instances when starting a new group instance"),
        SchedulerTest("clone-recover-no-shuffle-3",
                      "Don't shuffle instances when starting a new bundle instance"),
        SchedulerTest("clone-recover-no-shuffle-4",
                      "Don't shuffle instances when starting a new primitive instance with location preference"),
        SchedulerTest("clone-recover-no-shuffle-5",
                      "Don't shuffle instances when starting a new group instance with location preference"),
        SchedulerTest("clone-recover-no-shuffle-6",
                      "Don't shuffle instances when starting a new bundle instance with location preference"),
        SchedulerTest("clone-recover-no-shuffle-7",
                      "Don't shuffle instances when starting a new primitive instance that will be promoted"),
        SchedulerTest("clone-recover-no-shuffle-8",
                      "Don't shuffle instances when starting a new group instance that will be promoted"),
        SchedulerTest("clone-recover-no-shuffle-9",
                      "Don't shuffle instances when starting a new bundle instance that will be promoted"),
        SchedulerTest("clone-recover-no-shuffle-10",
                      "Don't shuffle instances when starting a new primitive instance that won't be promoted"),
        SchedulerTest("clone-recover-no-shuffle-11",
                      "Don't shuffle instances when starting a new group instance that won't be promoted"),
        SchedulerTest("clone-recover-no-shuffle-12",
                      "Don't shuffle instances when starting a new bundle instance that won't be promoted"),
        SchedulerTest("clone-max-zero", "Orphan processing with clone-max=0"),
        SchedulerTest("clone-anon-dup",
                      "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"),
        SchedulerTest("bug-lf-2160", "Don't shuffle clones due to colocation"),
        SchedulerTest("bug-lf-2213", "clone-node-max enforcement for cloned groups"),
        SchedulerTest("bug-lf-2153", "Clone ordering constraints"),
        SchedulerTest("bug-lf-2361", "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"),
        SchedulerTest("bug-lf-2317", "Avoid needless restart of primitive depending on a clone"),
        SchedulerTest("bug-lf-2453", "Enforce mandatory clone ordering without colocation"),
        SchedulerTest("bug-lf-2508", "Correctly reconstruct the status of anonymous cloned groups"),
        SchedulerTest("bug-lf-2544", "Balanced clone placement"),
        SchedulerTest("bug-lf-2445", "Redistribute clones with node-max > 1 and stickiness = 0"),
        SchedulerTest("bug-lf-2574", "Avoid clone shuffle"),
        SchedulerTest("bug-lf-2581", "Avoid group restart due to unrelated clone (re)start"),
        SchedulerTest("bug-cl-5168", "Don't shuffle clones"),
        SchedulerTest("bug-cl-5170", "Prevent clone from starting with on-fail=block"),
        SchedulerTest("clone-fail-block-colocation", "Move colocated group when failed clone has on-fail=block"),
        SchedulerTest("clone-interleave-1",
                      "Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)"),
        SchedulerTest("clone-interleave-2", "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)"),
        SchedulerTest("clone-interleave-3",
                      "Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)"),
        SchedulerTest("rebalance-unique-clones", "Rebalance unique clone instances with no stickiness"),
        SchedulerTest("clone-requires-quorum-recovery", "Clone with requires=quorum on failed node needing recovery"),
        SchedulerTest("clone-requires-quorum",
                      "Clone with requires=quorum with presumed-inactive instance on failed node"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("cloned_start_one", "order first clone then clone... first clone_min=2"),
        SchedulerTest("cloned_start_two", "order first clone then clone... first clone_min=2"),
        SchedulerTest("cloned_stop_one", "order first clone then clone... first clone_min=2"),
        SchedulerTest("cloned_stop_two", "order first clone then clone... first clone_min=2"),
        SchedulerTest("clone_min_interleave_start_one",
                      "order first clone then clone... first clone_min=2 and then has interleave=true"),
        SchedulerTest("clone_min_interleave_start_two",
                      "order first clone then clone... first clone_min=2 and then has interleave=true"),
        SchedulerTest("clone_min_interleave_stop_one",
                      "order first clone then clone... first clone_min=2 and then has interleave=true"),
        SchedulerTest("clone_min_interleave_stop_two",
                      "order first clone then clone... first clone_min=2 and then has interleave=true"),
        SchedulerTest("clone_min_start_one", "order first clone then primitive... first clone_min=2"),
        SchedulerTest("clone_min_start_two", "order first clone then primitive... first clone_min=2"),
        SchedulerTest("clone_min_stop_all", "order first clone then primitive... first clone_min=2"),
        SchedulerTest("clone_min_stop_one", "order first clone then primitive... first clone_min=2"),
        SchedulerTest("clone_min_stop_two", "order first clone then primitive... first clone_min=2"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("unfence-startup", "Clean unfencing"),
        SchedulerTest("unfence-definition", "Unfencing when the agent changes"),
        SchedulerTest("unfence-parameters", "Unfencing when the agent parameters changes"),
        SchedulerTest("unfence-device", "Unfencing when a cluster has only fence devices"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("promoted-0", "Stopped -> Unpromoted"),
        SchedulerTest("promoted-1", "Stopped -> Promote"),
        SchedulerTest("promoted-2", "Stopped -> Promote : notify"),
        SchedulerTest("promoted-3", "Stopped -> Promote : promoted location"),
        SchedulerTest("promoted-4", "Started -> Promote : promoted location"),
        SchedulerTest("promoted-5", "Promoted -> Promoted"),
        SchedulerTest("promoted-6", "Promoted -> Promoted (2)"),
        SchedulerTest("promoted-7", "Promoted -> Fenced"),
        SchedulerTest("promoted-8", "Promoted -> Fenced -> Moved"),
        SchedulerTest("promoted-9", "Stopped + Promotable + No quorum"),
        SchedulerTest("promoted-10", "Stopped -> Promotable : notify with monitor"),
        SchedulerTest("promoted-11", "Stopped -> Promote : colocation"),
        SchedulerTest("novell-239082", "Demote/Promote ordering"),
        SchedulerTest("novell-239087", "Stable promoted placement"),
        SchedulerTest("promoted-12", "Promotion based solely on rsc_location constraints"),
        SchedulerTest("promoted-13", "Include preferences of colocated resources when placing promoted"),
        SchedulerTest("promoted-demote", "Ordering when actions depends on demoting an unpromoted resource"),
        SchedulerTest("promoted-ordering", "Prevent resources from starting that need a promoted"),
        SchedulerTest("bug-1765", "Verify promoted-with-promoted colocation does not stop unpromoted instances"),
        SchedulerTest("promoted-group", "Promotion of cloned groups"),
        SchedulerTest("bug-lf-1852", "Don't shuffle promotable instances unnecessarily"),
        SchedulerTest("promoted-failed-demote", "Don't retry failed demote actions"),
        SchedulerTest("promoted-failed-demote-2", "Don't retry failed demote actions (notify=false)"),
        SchedulerTest("promoted-depend",
                      "Ensure resources that depend on promoted instance don't get allocated until that does"),
        SchedulerTest("promoted-reattach", "Re-attach to a running promoted"),
        SchedulerTest("promoted-allow-start", "Don't include promoted score if it would prevent allocation"),
        SchedulerTest("promoted-colocation",
                      "Allow promoted instances placemaker to be influenced by colocation constraints"),
        SchedulerTest("promoted-pseudo", "Make sure promote/demote pseudo actions are created correctly"),
        SchedulerTest("promoted-role", "Prevent target-role from promoting more than promoted-max instances"),
        SchedulerTest("bug-lf-2358", "Anti-colocation of promoted instances"),
        SchedulerTest("promoted-promotion-constraint", "Mandatory promoted colocation constraints"),
        SchedulerTest("unmanaged-promoted", "Ensure role is preserved for unmanaged resources"),
        SchedulerTest("promoted-unmanaged-monitor", "Start correct monitor for unmanaged promoted instances"),
        SchedulerTest("promoted-demote-2", "Demote does not clear past failure"),
        SchedulerTest("promoted-move", "Move promoted based on failure of colocated group"),
        SchedulerTest("promoted-probed-score", "Observe the promotion score of probed resources"),
        SchedulerTest("colocation_constraint_stops_promoted",
                      "cl#5054 - Ensure promoted is demoted when stopped by colocation constraint"),
        SchedulerTest("colocation_constraint_stops_unpromoted",
                      "cl#5054 - Ensure unpromoted is not demoted when stopped by colocation constraint"),
        SchedulerTest("order_constraint_stops_promoted",
                      "cl#5054 - Ensure promoted is demoted when stopped by order constraint"),
        SchedulerTest("order_constraint_stops_unpromoted",
                      "cl#5054 - Ensure unpromoted is not demoted when stopped by order constraint"),
        SchedulerTest("promoted_monitor_restart", "cl#5072 - Ensure promoted monitor operation will start after promotion"),
        SchedulerTest("bug-rh-880249", "Handle replacement of an m/s resource with a primitive"),
        SchedulerTest("bug-5143-ms-shuffle", "Prevent promoted instance shuffling due to promotion score"),
        SchedulerTest("promoted-demote-block", "Block promotion if demote fails with on-fail=block"),
        SchedulerTest("promoted-dependent-ban",
                      "Don't stop instances from being active because a dependent is banned from that host"),
        SchedulerTest("promoted-stop", "Stop instances due to location constraint with role=Started"),
        SchedulerTest("promoted-partially-demoted-group", "Allow partially demoted group to finish demoting"),
        SchedulerTest("bug-cl-5213", "Ensure role colocation with -INFINITY is enforced"),
        SchedulerTest("bug-cl-5219", "Allow unrelated resources with a common colocation target to remain promoted"),
        SchedulerTest("promoted-asymmetrical-order",
                      "Fix the behaviors of multi-state resources with asymmetrical ordering"),
        SchedulerTest("promoted-notify", "Promotion with notifications"),
        SchedulerTest("promoted-score-startup", "Use permanent promoted scores without LRM history"),
        SchedulerTest("failed-demote-recovery", "Recover resource in unpromoted role after demote fails"),
        SchedulerTest("failed-demote-recovery-promoted", "Recover resource in promoted role after demote fails"),
        SchedulerTest("on_fail_demote1", "Recovery with on-fail=\"demote\" on healthy cluster, remote, guest, and bundle nodes"),
        SchedulerTest("on_fail_demote2", "Recovery with on-fail=\"demote\" with promotion on different node"),
        SchedulerTest("on_fail_demote3", "Recovery with on-fail=\"demote\" with no promotion"),
        SchedulerTest("on_fail_demote4", "Recovery with on-fail=\"demote\" on failed cluster, remote, guest, and bundle nodes"),
        SchedulerTest("no_quorum_demote", "Promotable demotion and primitive stop with no-quorum-policy=\"demote\""),
        SchedulerTest("no-promote-on-unrunnable-guest", "Don't select bundle instance for promotion when container can't run"),
        SchedulerTest("leftover-pending-monitor", "Prevent a leftover pending monitor from causing unexpected stop of other instances"),
        SchedulerTest("promotable-colocation-with-node-attribute", "Promote dependent clone on nodes belonging to a site that has a primary clone promoted"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("history-1", "Correctly parse stateful-1 resource state"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("managed-0", "Managed (reference)"),
        SchedulerTest("managed-1", "Not managed - down"),
        SchedulerTest("managed-2", "Not managed - up"),
        SchedulerTest("bug-5028", "Shutdown should block if anything depends on an unmanaged resource"),
        SchedulerTest("bug-5028-detach", "Ensure detach still works"),
        SchedulerTest("bug-5028-bottom",
                      "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack"),
        SchedulerTest("unmanaged-stop-1",
                      "cl#5155 - Block the stop of resources if any depending resource is unmanaged"),
        SchedulerTest("unmanaged-stop-2",
                      "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged"),
        SchedulerTest("unmanaged-stop-3",
                      "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged"),
        SchedulerTest("unmanaged-stop-4",
                      "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged"),
        SchedulerTest("unmanaged-block-restart",
                      "Block restart of resources if any dependent resource in a group is unmanaged"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("interleave-0", "Interleave (reference)"),
        SchedulerTest("interleave-1", "coloc - not interleaved"),
        SchedulerTest("interleave-2", "coloc - interleaved"),
        SchedulerTest("interleave-3", "coloc - interleaved (2)"),
        SchedulerTest("interleave-pseudo-stop", "Interleaved clone during stonith"),
        SchedulerTest("interleave-stop", "Interleaved clone during stop"),
        SchedulerTest("interleave-restart", "Interleaved clone during dependency restart"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("notify-0", "Notify reference"),
        SchedulerTest("notify-1", "Notify simple"),
        SchedulerTest("notify-2", "Notify simple, confirm"),
        SchedulerTest("notify-3", "Notify move, confirm"),
        SchedulerTest("novell-239079", "Notification priority"),
        SchedulerTest("notifs-for-unrunnable", "Don't schedule notifications for an unrunnable action"),
        SchedulerTest("route-remote-notify", "Route remote notify actions through correct cluster node"),
        SchedulerTest("notify-behind-stopping-remote", "Don't schedule notifications behind stopped remote"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("594", "OSDL #594 - Unrunnable actions scheduled in transition"),
        SchedulerTest("662", "OSDL #662 - Two resources start on one node when incarnation_node_max = 1"),
        SchedulerTest("696", "OSDL #696 - CRM starts stonith RA without monitor"),
        SchedulerTest("726", "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop"),
        SchedulerTest("735", "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3"),
        SchedulerTest("764", "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1"),
        SchedulerTest("797", "OSDL #797 - Assert triggered: task_id_i > max_call_id"),
        SchedulerTest("829", "OSDL #829"),
        SchedulerTest("994",
                      "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted"),
        SchedulerTest("994-2", "OSDL #994 - with a dependent resource"),
        SchedulerTest("1360", "OSDL #1360 - Clone stickiness"),
        SchedulerTest("1484", "OSDL #1484 - on_fail=stop"),
        SchedulerTest("1494", "OSDL #1494 - Clone stability"),
        SchedulerTest("unrunnable-1", "Unrunnable"),
        SchedulerTest("unrunnable-2", "Unrunnable 2"),
        SchedulerTest("stonith-0", "Stonith loop - 1"),
        SchedulerTest("stonith-1", "Stonith loop - 2"),
        SchedulerTest("stonith-2", "Stonith loop - 3"),
        SchedulerTest("stonith-3", "Stonith startup"),
        SchedulerTest("stonith-4", "Stonith node state"),
        SchedulerTest("dc-fence-ordering", "DC needs fencing while other nodes are shutting down"),
        SchedulerTest("bug-1572-1", "Recovery of groups depending on promotable role"),
        SchedulerTest("bug-1572-2", "Recovery of groups depending on promotable role when promoted is not re-promoted"),
        SchedulerTest("bug-1685", "Depends-on-promoted ordering"),
        SchedulerTest("bug-1822", "Don't promote partially active groups"),
        SchedulerTest("bug-pm-11", "New resource added to a m/s group"),
        SchedulerTest("bug-pm-12", "Recover only the failed portion of a cloned group"),
        SchedulerTest("bug-n-387749", "Don't shuffle clone instances"),
        SchedulerTest("bug-n-385265",
                      "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"),
        SchedulerTest("bug-n-385265-2",
                      "Ensure groups are migrated instead of remaining partially active on the current node"),
        SchedulerTest("bug-lf-1920", "Correctly handle probes that find active resources"),
        SchedulerTest("bnc-515172", "Location constraint with multiple expressions"),
        SchedulerTest("colocate-primitive-with-clone", "Optional colocation with a clone"),
        SchedulerTest("use-after-free-merge", "Use-after-free in native_merge_weights"),
        SchedulerTest("bug-lf-2551", "STONITH ordering for stop"),
        SchedulerTest("bug-lf-2606", "Stonith implies demote"),
        SchedulerTest("bug-lf-2474", "Ensure resource op timeout takes precedence over op_defaults"),
        SchedulerTest("bug-suse-707150", "Prevent vm-01 from starting due to colocation/ordering"),
        SchedulerTest("bug-5014-A-start-B-start", "Verify when A starts B starts using symmetrical=false"),
        SchedulerTest("bug-5014-A-stop-B-started",
                      "Verify when A stops B does not stop if it has already started using symmetric=false"),
        SchedulerTest("bug-5014-A-stopped-B-stopped",
                      "Verify when A is stopped and B has not started, B does not start before A using symmetric=false"),
        SchedulerTest("bug-5014-CthenAthenB-C-stopped",
                      "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts"),
        SchedulerTest("bug-5014-CLONE-A-start-B-start",
                      "Verify when A starts B starts using clone resources with symmetric=false"),
        SchedulerTest("bug-5014-CLONE-A-stop-B-started",
                      "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false"),
        SchedulerTest("bug-5014-GROUP-A-start-B-start",
                      "Verify when A starts B starts when using group resources with symmetric=false"),
        SchedulerTest("bug-5014-GROUP-A-stopped-B-started",
                      "Verify when A stops B does not stop if it has already started using group resources with symmetric=false"),
        SchedulerTest("bug-5014-GROUP-A-stopped-B-stopped",
                      "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false"),
        SchedulerTest("bug-5014-ordered-set-symmetrical-false",
                      "Verify ordered sets work with symmetrical=false"),
        SchedulerTest("bug-5014-ordered-set-symmetrical-true",
                      "Verify ordered sets work with symmetrical=true"),
        SchedulerTest("clbz5007-promotable-colocation",
                      "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources"),
        SchedulerTest("bug-5038", "Prevent restart of anonymous clones when clone-max decreases"),
        SchedulerTest("bug-5025-1", "Automatically clean up failcount after resource config change with reload"),
        SchedulerTest("bug-5025-2", "Make sure clear failcount action isn't set when config does not change"),
        SchedulerTest("bug-5025-3", "Automatically clean up failcount after resource config change with restart"),
        SchedulerTest("bug-5025-4", "Clear failcount when last failure is a start op and rsc attributes changed"),
        SchedulerTest("failcount", "Ensure failcounts are correctly expired"),
        SchedulerTest("failcount-block", "Ensure failcounts are not expired when on-fail=block is present"),
        SchedulerTest("per-op-failcount", "Ensure per-operation failcount is handled and not passed to fence agent"),
        SchedulerTest("on-fail-ignore", "Ensure on-fail=ignore works even beyond migration-threshold"),
        SchedulerTest("monitor-onfail-restart", "bug-5058 - Monitor failure with on-fail set to restart"),
        SchedulerTest("monitor-onfail-stop", "bug-5058 - Monitor failure wiht on-fail set to stop"),
        SchedulerTest("bug-5059", "No need to restart p_stateful1:*"),
        SchedulerTest("bug-5069-op-enabled", "Test on-fail=ignore with failure when monitor is enabled"),
        SchedulerTest("bug-5069-op-disabled", "Test on-fail-ignore with failure when monitor is disabled"),
        SchedulerTest("obsolete-lrm-resource", "cl#5115 - Do not use obsolete lrm_resource sections"),
        SchedulerTest("expire-non-blocked-failure",
                      "Ignore failure-timeout only if the failed operation has on-fail=block"),
        SchedulerTest("asymmetrical-order-move", "Respect asymmetrical ordering when trying to move resources"),
        SchedulerTest("asymmetrical-order-restart", "Respect asymmetrical ordering when restarting dependent resource"),
        SchedulerTest("start-then-stop-with-unfence", "Avoid graph loop with start-then-stop constraint plus unfencing"),
        SchedulerTest("order-expired-failure", "Order failcount cleanup after remote fencing"),
        SchedulerTest("expired-stop-1", "Expired stop failure should not block resource"),
        SchedulerTest("ignore_stonith_rsc_order1",
                      "cl#5056- Ignore order constraint between stonith and non-stonith rsc"),
        SchedulerTest("ignore_stonith_rsc_order2",
                      "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith"),
        SchedulerTest("ignore_stonith_rsc_order3", "cl#5056- Ignore order constraint, stonith clone and mixed group"),
        SchedulerTest("ignore_stonith_rsc_order4",
                      "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group"),
        SchedulerTest("honor_stonith_rsc_order1",
                      "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)"),
        SchedulerTest("honor_stonith_rsc_order2",
                      "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)"),
        SchedulerTest("honor_stonith_rsc_order3",
                      "cl#5056- Honor order constraint, stonith clones with nested pure stonith group"),
        SchedulerTest("honor_stonith_rsc_order4",
                      "cl#5056- Honor order constraint, between two native stonith rscs"),
        SchedulerTest("multiply-active-stonith", "Multiply active stonith"),
        SchedulerTest("probe-timeout", "cl#5099 - Default probe timeout"),
        SchedulerTest("order-first-probes",
                      "cl#5301 - respect order constraints when relevant resources are being probed"),
        SchedulerTest("concurrent-fencing", "Allow performing fencing operations in parallel"),
        SchedulerTest("priority-fencing-delay", "Delay fencing targeting the more significant node"),
        SchedulerTest("pending-node-no-uname", "Do not fence a pending node that doesn't have an uname in node state yet"),
        SchedulerTest("node-pending-timeout", "Fence a pending node that has reached `node-pending-timeout`"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("systemhealth1", "System Health ()               #1"),
        SchedulerTest("systemhealth2", "System Health ()               #2"),
        SchedulerTest("systemhealth3", "System Health ()               #3"),
        SchedulerTest("systemhealthn1", "System Health (None)           #1"),
        SchedulerTest("systemhealthn2", "System Health (None)           #2"),
        SchedulerTest("systemhealthn3", "System Health (None)           #3"),
        SchedulerTest("systemhealthm1", "System Health (Migrate On Red) #1"),
        SchedulerTest("systemhealthm2", "System Health (Migrate On Red) #2"),
        SchedulerTest("systemhealthm3", "System Health (Migrate On Red) #3"),
        SchedulerTest("systemhealtho1", "System Health (Only Green)     #1"),
        SchedulerTest("systemhealtho2", "System Health (Only Green)     #2"),
        SchedulerTest("systemhealtho3", "System Health (Only Green)     #3"),
        SchedulerTest("systemhealthp1", "System Health (Progessive)     #1"),
        SchedulerTest("systemhealthp2", "System Health (Progessive)     #2"),
        SchedulerTest("systemhealthp3", "System Health (Progessive)     #3"),
        SchedulerTest("allow-unhealthy-nodes", "System Health (migrate-on-red + allow-unhealth-nodes)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("utilization", "Placement Strategy - utilization"),
        SchedulerTest("minimal", "Placement Strategy - minimal"),
        SchedulerTest("balanced", "Placement Strategy - balanced"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("placement-stickiness", "Optimized Placement Strategy - stickiness"),
        SchedulerTest("placement-priority", "Optimized Placement Strategy - priority"),
        SchedulerTest("placement-location", "Optimized Placement Strategy - location"),
        SchedulerTest("placement-capacity", "Optimized Placement Strategy - capacity"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("utilization-order1", "Utilization Order - Simple"),
        SchedulerTest("utilization-order2", "Utilization Order - Complex"),
        SchedulerTest("utilization-order3", "Utilization Order - Migrate"),
        SchedulerTest("utilization-order4", "Utilization Order - Live Migration (bnc#695440)"),
        SchedulerTest("utilization-complex", "Utilization with complex relationships"),
        SchedulerTest("utilization-shuffle",
                      "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3"),
        SchedulerTest("load-stopped-loop", "Avoid transition loop due to load_stopped (cl#5044)"),
        SchedulerTest("load-stopped-loop-2",
                      "cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("colocated-utilization-primitive-1", "Colocated Utilization - Primitive"),
        SchedulerTest("colocated-utilization-primitive-2", "Colocated Utilization - Choose the most capable node"),
        SchedulerTest("colocated-utilization-group", "Colocated Utilization - Group"),
        SchedulerTest("colocated-utilization-clone", "Colocated Utilization - Clone"),
        SchedulerTest("utilization-check-allowed-nodes",
                      "Only check the capacities of the nodes that can run the resource"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("node-maintenance-1", "cl#5128 - Node maintenance"),
        SchedulerTest("node-maintenance-2", "cl#5128 - Node maintenance (coming out of maintenance mode)"),
        SchedulerTest("shutdown-maintenance-node", "Do not fence a maintenance node if it shuts down cleanly"),
        SchedulerTest("rsc-maintenance", "Per-resource maintenance"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("not-installed-agent", "The resource agent is missing"),
        SchedulerTest("not-installed-tools", "Something the resource agent needs is missing"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("stopped-monitor-00", "Stopped Monitor - initial start"),
        SchedulerTest("stopped-monitor-01", "Stopped Monitor - failed started"),
        SchedulerTest("stopped-monitor-02", "Stopped Monitor - started multi-up"),
        SchedulerTest("stopped-monitor-03", "Stopped Monitor - stop started"),
        SchedulerTest("stopped-monitor-04", "Stopped Monitor - failed stop"),
        SchedulerTest("stopped-monitor-05", "Stopped Monitor - start unmanaged"),
        SchedulerTest("stopped-monitor-06", "Stopped Monitor - unmanaged multi-up"),
        SchedulerTest("stopped-monitor-07", "Stopped Monitor - start unmanaged multi-up"),
        SchedulerTest("stopped-monitor-08", "Stopped Monitor - migrate"),
        SchedulerTest("stopped-monitor-09", "Stopped Monitor - unmanage started"),
        SchedulerTest("stopped-monitor-10", "Stopped Monitor - unmanaged started multi-up"),
        SchedulerTest("stopped-monitor-11", "Stopped Monitor - stop unmanaged started"),
        SchedulerTest("stopped-monitor-12", "Stopped Monitor - unmanaged started multi-up (target-role=Stopped)"),
        SchedulerTest("stopped-monitor-20", "Stopped Monitor - initial stop"),
        SchedulerTest("stopped-monitor-21", "Stopped Monitor - stopped single-up"),
        SchedulerTest("stopped-monitor-22", "Stopped Monitor - stopped multi-up"),
        SchedulerTest("stopped-monitor-23", "Stopped Monitor - start stopped"),
        SchedulerTest("stopped-monitor-24", "Stopped Monitor - unmanage stopped"),
        SchedulerTest("stopped-monitor-25", "Stopped Monitor - unmanaged stopped multi-up"),
        SchedulerTest("stopped-monitor-26", "Stopped Monitor - start unmanaged stopped"),
        SchedulerTest("stopped-monitor-27", "Stopped Monitor - unmanaged stopped multi-up (target-role=Started)"),
        SchedulerTest("stopped-monitor-30", "Stopped Monitor - new node started"),
        SchedulerTest("stopped-monitor-31", "Stopped Monitor - new node stopped"),
    ]),
    SchedulerTestGroup([
        # This is a combo test to check:
        # - probe timeout defaults to the minimum-interval monitor's
        # - duplicate recurring operations are ignored
        # - if timeout spec is bad, the default timeout is used
        # - failure is blocked with on-fail=block even if ISO8601 interval is specified
        # - started/stopped role monitors are started/stopped on right nodes
        SchedulerTest("intervals", "Recurring monitor interval handling"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("ticket-primitive-1", "Ticket - Primitive (loss-policy=stop, initial)"),
        SchedulerTest("ticket-primitive-2", "Ticket - Primitive (loss-policy=stop, granted)"),
        SchedulerTest("ticket-primitive-3", "Ticket - Primitive (loss-policy-stop, revoked)"),
        SchedulerTest("ticket-primitive-4", "Ticket - Primitive (loss-policy=demote, initial)"),
        SchedulerTest("ticket-primitive-5", "Ticket - Primitive (loss-policy=demote, granted)"),
        SchedulerTest("ticket-primitive-6", "Ticket - Primitive (loss-policy=demote, revoked)"),
        SchedulerTest("ticket-primitive-7", "Ticket - Primitive (loss-policy=fence, initial)"),
        SchedulerTest("ticket-primitive-8", "Ticket - Primitive (loss-policy=fence, granted)"),
        SchedulerTest("ticket-primitive-9", "Ticket - Primitive (loss-policy=fence, revoked)"),
        SchedulerTest("ticket-primitive-10", "Ticket - Primitive (loss-policy=freeze, initial)"),
        SchedulerTest("ticket-primitive-11", "Ticket - Primitive (loss-policy=freeze, granted)"),
        SchedulerTest("ticket-primitive-12", "Ticket - Primitive (loss-policy=freeze, revoked)"),
        SchedulerTest("ticket-primitive-13", "Ticket - Primitive (loss-policy=stop, standby, granted)"),
        SchedulerTest("ticket-primitive-14", "Ticket - Primitive (loss-policy=stop, granted, standby)"),
        SchedulerTest("ticket-primitive-15", "Ticket - Primitive (loss-policy=stop, standby, revoked)"),
        SchedulerTest("ticket-primitive-16", "Ticket - Primitive (loss-policy=demote, standby, granted)"),
        SchedulerTest("ticket-primitive-17", "Ticket - Primitive (loss-policy=demote, granted, standby)"),
        SchedulerTest("ticket-primitive-18", "Ticket - Primitive (loss-policy=demote, standby, revoked)"),
        SchedulerTest("ticket-primitive-19", "Ticket - Primitive (loss-policy=fence, standby, granted)"),
        SchedulerTest("ticket-primitive-20", "Ticket - Primitive (loss-policy=fence, granted, standby)"),
        SchedulerTest("ticket-primitive-21", "Ticket - Primitive (loss-policy=fence, standby, revoked)"),
        SchedulerTest("ticket-primitive-22", "Ticket - Primitive (loss-policy=freeze, standby, granted)"),
        SchedulerTest("ticket-primitive-23", "Ticket - Primitive (loss-policy=freeze, granted, standby)"),
        SchedulerTest("ticket-primitive-24", "Ticket - Primitive (loss-policy=freeze, standby, revoked)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("ticket-group-1", "Ticket - Group (loss-policy=stop, initial)"),
        SchedulerTest("ticket-group-2", "Ticket - Group (loss-policy=stop, granted)"),
        SchedulerTest("ticket-group-3", "Ticket - Group (loss-policy-stop, revoked)"),
        SchedulerTest("ticket-group-4", "Ticket - Group (loss-policy=demote, initial)"),
        SchedulerTest("ticket-group-5", "Ticket - Group (loss-policy=demote, granted)"),
        SchedulerTest("ticket-group-6", "Ticket - Group (loss-policy=demote, revoked)"),
        SchedulerTest("ticket-group-7", "Ticket - Group (loss-policy=fence, initial)"),
        SchedulerTest("ticket-group-8", "Ticket - Group (loss-policy=fence, granted)"),
        SchedulerTest("ticket-group-9", "Ticket - Group (loss-policy=fence, revoked)"),
        SchedulerTest("ticket-group-10", "Ticket - Group (loss-policy=freeze, initial)"),
        SchedulerTest("ticket-group-11", "Ticket - Group (loss-policy=freeze, granted)"),
        SchedulerTest("ticket-group-12", "Ticket - Group (loss-policy=freeze, revoked)"),
        SchedulerTest("ticket-group-13", "Ticket - Group (loss-policy=stop, standby, granted)"),
        SchedulerTest("ticket-group-14", "Ticket - Group (loss-policy=stop, granted, standby)"),
        SchedulerTest("ticket-group-15", "Ticket - Group (loss-policy=stop, standby, revoked)"),
        SchedulerTest("ticket-group-16", "Ticket - Group (loss-policy=demote, standby, granted)"),
        SchedulerTest("ticket-group-17", "Ticket - Group (loss-policy=demote, granted, standby)"),
        SchedulerTest("ticket-group-18", "Ticket - Group (loss-policy=demote, standby, revoked)"),
        SchedulerTest("ticket-group-19", "Ticket - Group (loss-policy=fence, standby, granted)"),
        SchedulerTest("ticket-group-20", "Ticket - Group (loss-policy=fence, granted, standby)"),
        SchedulerTest("ticket-group-21", "Ticket - Group (loss-policy=fence, standby, revoked)"),
        SchedulerTest("ticket-group-22", "Ticket - Group (loss-policy=freeze, standby, granted)"),
        SchedulerTest("ticket-group-23", "Ticket - Group (loss-policy=freeze, granted, standby)"),
        SchedulerTest("ticket-group-24", "Ticket - Group (loss-policy=freeze, standby, revoked)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("ticket-clone-1", "Ticket - Clone (loss-policy=stop, initial)"),
        SchedulerTest("ticket-clone-2", "Ticket - Clone (loss-policy=stop, granted)"),
        SchedulerTest("ticket-clone-3", "Ticket - Clone (loss-policy-stop, revoked)"),
        SchedulerTest("ticket-clone-4", "Ticket - Clone (loss-policy=demote, initial)"),
        SchedulerTest("ticket-clone-5", "Ticket - Clone (loss-policy=demote, granted)"),
        SchedulerTest("ticket-clone-6", "Ticket - Clone (loss-policy=demote, revoked)"),
        SchedulerTest("ticket-clone-7", "Ticket - Clone (loss-policy=fence, initial)"),
        SchedulerTest("ticket-clone-8", "Ticket - Clone (loss-policy=fence, granted)"),
        SchedulerTest("ticket-clone-9", "Ticket - Clone (loss-policy=fence, revoked)"),
        SchedulerTest("ticket-clone-10", "Ticket - Clone (loss-policy=freeze, initial)"),
        SchedulerTest("ticket-clone-11", "Ticket - Clone (loss-policy=freeze, granted)"),
        SchedulerTest("ticket-clone-12", "Ticket - Clone (loss-policy=freeze, revoked)"),
        SchedulerTest("ticket-clone-13", "Ticket - Clone (loss-policy=stop, standby, granted)"),
        SchedulerTest("ticket-clone-14", "Ticket - Clone (loss-policy=stop, granted, standby)"),
        SchedulerTest("ticket-clone-15", "Ticket - Clone (loss-policy=stop, standby, revoked)"),
        SchedulerTest("ticket-clone-16", "Ticket - Clone (loss-policy=demote, standby, granted)"),
        SchedulerTest("ticket-clone-17", "Ticket - Clone (loss-policy=demote, granted, standby)"),
        SchedulerTest("ticket-clone-18", "Ticket - Clone (loss-policy=demote, standby, revoked)"),
        SchedulerTest("ticket-clone-19", "Ticket - Clone (loss-policy=fence, standby, granted)"),
        SchedulerTest("ticket-clone-20", "Ticket - Clone (loss-policy=fence, granted, standby)"),
        SchedulerTest("ticket-clone-21", "Ticket - Clone (loss-policy=fence, standby, revoked)"),
        SchedulerTest("ticket-clone-22", "Ticket - Clone (loss-policy=freeze, standby, granted)"),
        SchedulerTest("ticket-clone-23", "Ticket - Clone (loss-policy=freeze, granted, standby)"),
        SchedulerTest("ticket-clone-24", "Ticket - Clone (loss-policy=freeze, standby, revoked)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("ticket-promoted-1", "Ticket - Promoted (loss-policy=stop, initial)"),
        SchedulerTest("ticket-promoted-2", "Ticket - Promoted (loss-policy=stop, granted)"),
        SchedulerTest("ticket-promoted-3", "Ticket - Promoted (loss-policy-stop, revoked)"),
        SchedulerTest("ticket-promoted-4", "Ticket - Promoted (loss-policy=demote, initial)"),
        SchedulerTest("ticket-promoted-5", "Ticket - Promoted (loss-policy=demote, granted)"),
        SchedulerTest("ticket-promoted-6", "Ticket - Promoted (loss-policy=demote, revoked)"),
        SchedulerTest("ticket-promoted-7", "Ticket - Promoted (loss-policy=fence, initial)"),
        SchedulerTest("ticket-promoted-8", "Ticket - Promoted (loss-policy=fence, granted)"),
        SchedulerTest("ticket-promoted-9", "Ticket - Promoted (loss-policy=fence, revoked)"),
        SchedulerTest("ticket-promoted-10", "Ticket - Promoted (loss-policy=freeze, initial)"),
        SchedulerTest("ticket-promoted-11", "Ticket - Promoted (loss-policy=freeze, granted)"),
        SchedulerTest("ticket-promoted-12", "Ticket - Promoted (loss-policy=freeze, revoked)"),
        SchedulerTest("ticket-promoted-13", "Ticket - Promoted (loss-policy=stop, standby, granted)"),
        SchedulerTest("ticket-promoted-14", "Ticket - Promoted (loss-policy=stop, granted, standby)"),
        SchedulerTest("ticket-promoted-15", "Ticket - Promoted (loss-policy=stop, standby, revoked)"),
        SchedulerTest("ticket-promoted-16", "Ticket - Promoted (loss-policy=demote, standby, granted)"),
        SchedulerTest("ticket-promoted-17", "Ticket - Promoted (loss-policy=demote, granted, standby)"),
        SchedulerTest("ticket-promoted-18", "Ticket - Promoted (loss-policy=demote, standby, revoked)"),
        SchedulerTest("ticket-promoted-19", "Ticket - Promoted (loss-policy=fence, standby, granted)"),
        SchedulerTest("ticket-promoted-20", "Ticket - Promoted (loss-policy=fence, granted, standby)"),
        SchedulerTest("ticket-promoted-21", "Ticket - Promoted (loss-policy=fence, standby, revoked)"),
        SchedulerTest("ticket-promoted-22", "Ticket - Promoted (loss-policy=freeze, standby, granted)"),
        SchedulerTest("ticket-promoted-23", "Ticket - Promoted (loss-policy=freeze, granted, standby)"),
        SchedulerTest("ticket-promoted-24", "Ticket - Promoted (loss-policy=freeze, standby, revoked)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("ticket-rsc-sets-1", "Ticket - Resource sets (1 ticket, initial)"),
        SchedulerTest("ticket-rsc-sets-2", "Ticket - Resource sets (1 ticket, granted)"),
        SchedulerTest("ticket-rsc-sets-3", "Ticket - Resource sets (1 ticket, revoked)"),
        SchedulerTest("ticket-rsc-sets-4", "Ticket - Resource sets (2 tickets, initial)"),
        SchedulerTest("ticket-rsc-sets-5", "Ticket - Resource sets (2 tickets, granted)"),
        SchedulerTest("ticket-rsc-sets-6", "Ticket - Resource sets (2 tickets, granted)"),
        SchedulerTest("ticket-rsc-sets-7", "Ticket - Resource sets (2 tickets, revoked)"),
        SchedulerTest("ticket-rsc-sets-8", "Ticket - Resource sets (1 ticket, standby, granted)"),
        SchedulerTest("ticket-rsc-sets-9", "Ticket - Resource sets (1 ticket, granted, standby)"),
        SchedulerTest("ticket-rsc-sets-10", "Ticket - Resource sets (1 ticket, standby, revoked)"),
        SchedulerTest("ticket-rsc-sets-11", "Ticket - Resource sets (2 tickets, standby, granted)"),
        SchedulerTest("ticket-rsc-sets-12", "Ticket - Resource sets (2 tickets, standby, granted)"),
        SchedulerTest("ticket-rsc-sets-13", "Ticket - Resource sets (2 tickets, granted, standby)"),
        SchedulerTest("ticket-rsc-sets-14", "Ticket - Resource sets (2 tickets, standby, revoked)"),
        SchedulerTest("cluster-specific-params", "Cluster-specific instance attributes based on rules"),
        SchedulerTest("site-specific-params", "Site-specific instance attributes based on rules"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("template-1", "Template - 1"),
        SchedulerTest("template-2", "Template - 2"),
        SchedulerTest("template-3", "Template - 3 (merge operations)"),
        SchedulerTest("template-coloc-1", "Template - Colocation 1"),
        SchedulerTest("template-coloc-2", "Template - Colocation 2"),
        SchedulerTest("template-coloc-3", "Template - Colocation 3"),
        SchedulerTest("template-order-1", "Template - Order 1"),
        SchedulerTest("template-order-2", "Template - Order 2"),
        SchedulerTest("template-order-3", "Template - Order 3"),
        SchedulerTest("template-ticket", "Template - Ticket"),
        SchedulerTest("template-rsc-sets-1", "Template - Resource Sets 1"),
        SchedulerTest("template-rsc-sets-2", "Template - Resource Sets 2"),
        SchedulerTest("template-rsc-sets-3", "Template - Resource Sets 3"),
        SchedulerTest("template-rsc-sets-4", "Template - Resource Sets 4"),
        SchedulerTest("template-clone-primitive", "Cloned primitive from template"),
        SchedulerTest("template-clone-group", "Cloned group from template"),
        SchedulerTest("location-sets-templates", "Resource sets and templates - Location"),
        SchedulerTest("tags-coloc-order-1", "Tags - Colocation and Order (Simple)"),
        SchedulerTest("tags-coloc-order-2", "Tags - Colocation and Order (Resource Sets with Templates)"),
        SchedulerTest("tags-location", "Tags - Location"),
        SchedulerTest("tags-ticket", "Tags - Ticket"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("container-1", "Container - initial"),
        SchedulerTest("container-2", "Container - monitor failed"),
        SchedulerTest("container-3", "Container - stop failed"),
        SchedulerTest("container-4", "Container - reached migration-threshold"),
        SchedulerTest("container-group-1", "Container in group - initial"),
        SchedulerTest("container-group-2", "Container in group - monitor failed"),
        SchedulerTest("container-group-3", "Container in group - stop failed"),
        SchedulerTest("container-group-4", "Container in group - reached migration-threshold"),
        SchedulerTest("container-is-remote-node", "Place resource within container when container is remote-node"),
        SchedulerTest("bug-rh-1097457", "Kill user defined container/contents ordering"),
        SchedulerTest("bug-cl-5247", "Graph loop when recovering m/s resource in a container"),
        SchedulerTest("bundle-order-startup", "Bundle startup ordering"),
        SchedulerTest("bundle-order-partial-start",
                      "Bundle startup ordering when some dependencies are already running"),
        SchedulerTest("bundle-order-partial-start-2",
                      "Bundle startup ordering when some dependencies and the container are already running"),
        SchedulerTest("bundle-order-stop", "Bundle stop ordering"),
        SchedulerTest("bundle-order-partial-stop", "Bundle startup ordering when some dependencies are already stopped"),
        SchedulerTest("bundle-order-stop-on-remote", "Stop nested resource after bringing up the connection"),
        SchedulerTest("bundle-order-startup-clone", "Prevent startup because bundle isn't promoted"),
        SchedulerTest("bundle-order-startup-clone-2", "Bundle startup with clones"),
        SchedulerTest("bundle-order-stop-clone", "Stop bundle because clone is stopping"),
        SchedulerTest("bundle-interleave-start", "Interleave bundle starts"),
        SchedulerTest("bundle-interleave-promote", "Interleave bundle promotes"),
        SchedulerTest("bundle-nested-colocation", "Colocation of nested connection resources"),
        SchedulerTest("bundle-order-fencing",
                      "Order pseudo bundle fencing after parent node fencing if both are happening"),
        SchedulerTest("bundle-probe-order-1", "order 1"),
        SchedulerTest("bundle-probe-order-2", "order 2"),
        SchedulerTest("bundle-probe-order-3", "order 3"),
        SchedulerTest("bundle-probe-remotes", "Ensure remotes get probed too"),
        SchedulerTest("bundle-replicas-change", "Change bundle from 1 replica to multiple"),
        SchedulerTest("bundle-connection-with-container", "Don't move a container due to connection preferences"),
        SchedulerTest("nested-remote-recovery", "Recover bundle's container hosted on remote node"),
        SchedulerTest("bundle-promoted-location-1",
                      "Promotable bundle, positive location"),
        SchedulerTest("bundle-promoted-location-2",
                      "Promotable bundle, negative location"),
        SchedulerTest("bundle-promoted-location-3",
                      "Promotable bundle, positive location for promoted role"),
        SchedulerTest("bundle-promoted-location-4",
                      "Promotable bundle, negative location for promoted role"),
        SchedulerTest("bundle-promoted-location-5",
                      "Promotable bundle, positive location for unpromoted role"),
        SchedulerTest("bundle-promoted-location-6",
                      "Promotable bundle, negative location for unpromoted role"),
        SchedulerTest("bundle-promoted-colocation-1",
                      "Primary promoted bundle, dependent primitive (mandatory coloc)"),
        SchedulerTest("bundle-promoted-colocation-2",
                      "Primary promoted bundle, dependent primitive (optional coloc)"),
        SchedulerTest("bundle-promoted-colocation-3",
                      "Dependent promoted bundle, primary primitive (mandatory coloc)"),
        SchedulerTest("bundle-promoted-colocation-4",
                      "Dependent promoted bundle, primary primitive (optional coloc)"),
        SchedulerTest("bundle-promoted-colocation-5",
                      "Primary and dependent promoted bundle instances (mandatory coloc)"),
        SchedulerTest("bundle-promoted-colocation-6",
                      "Primary and dependent promoted bundle instances (optional coloc)"),
        SchedulerTest("bundle-promoted-anticolocation-1",
                      "Primary promoted bundle, dependent primitive (mandatory anti)"),
        SchedulerTest("bundle-promoted-anticolocation-2",
                      "Primary promoted bundle, dependent primitive (optional anti)"),
        SchedulerTest("bundle-promoted-anticolocation-3",
                      "Dependent promoted bundle, primary primitive (mandatory anti)"),
        SchedulerTest("bundle-promoted-anticolocation-4",
                      "Dependent promoted bundle, primary primitive (optional anti)"),
        SchedulerTest("bundle-promoted-anticolocation-5",
                      "Primary and dependent promoted bundle instances (mandatory anti)"),
        SchedulerTest("bundle-promoted-anticolocation-6",
                      "Primary and dependent promoted bundle instances (optional anti)"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("whitebox-fail1", "Fail whitebox container rsc"),
        SchedulerTest("whitebox-fail2", "Fail cluster connection to guest node"),
        SchedulerTest("whitebox-fail3", "Failed containers should not run nested on remote nodes"),
        SchedulerTest("whitebox-start", "Start whitebox container with resources assigned to it"),
        SchedulerTest("whitebox-stop", "Stop whitebox container with resources assigned to it"),
        SchedulerTest("whitebox-move", "Move whitebox container with resources assigned to it"),
        SchedulerTest("whitebox-asymmetric", "Verify connection rsc opts-in based on container resource"),
        SchedulerTest("whitebox-ms-ordering", "Verify promote/demote can not occur before connection is established"),
        SchedulerTest("whitebox-ms-ordering-move", "Stop/Start cycle within a moving container"),
        SchedulerTest("whitebox-orphaned", "Properly shutdown orphaned whitebox container"),
        SchedulerTest("whitebox-orphan-ms", "Properly tear down orphan ms resources on remote-nodes"),
        SchedulerTest("whitebox-unexpectedly-running", "Recover container nodes the cluster did not start"),
        SchedulerTest("whitebox-migrate1", "Migrate both container and connection resource"),
        SchedulerTest("whitebox-imply-stop-on-fence",
                      "imply stop action on container node rsc when host node is fenced"),
        SchedulerTest("whitebox-nested-group", "Verify guest remote-node works nested in a group"),
        SchedulerTest("guest-node-host-dies", "Verify guest node is recovered if host goes away"),
        SchedulerTest("guest-node-cleanup", "Order guest node connection recovery after container probe"),
        SchedulerTest("guest-host-not-fenceable", "Actions on guest node are unrunnable if host is unclean and cannot be fenced"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("remote-startup-probes", "Baremetal remote-node startup probes"),
        SchedulerTest("remote-startup", "Startup a newly discovered remote-nodes with no status"),
        SchedulerTest("remote-fence-unclean", "Fence unclean baremetal remote-node"),
        SchedulerTest("remote-fence-unclean2",
                      "Fence baremetal remote-node after cluster node fails and connection can not be recovered"),
        SchedulerTest("remote-fence-unclean-3", "Probe failed remote nodes (triggers fencing)"),
        SchedulerTest("remote-move", "Move remote-node connection resource"),
        SchedulerTest("remote-disable", "Disable a baremetal remote-node"),
        SchedulerTest("remote-probe-disable", "Probe then stop a baremetal remote-node"),
        SchedulerTest("remote-orphaned", "Properly shutdown orphaned connection resource"),
        SchedulerTest("remote-orphaned2",
                      "verify we can handle orphaned remote connections with active resources on the remote"),
        SchedulerTest("remote-recover", "Recover connection resource after cluster-node fails"),
        SchedulerTest("remote-stale-node-entry",
                      "Make sure we properly handle leftover remote-node entries in the node section"),
        SchedulerTest("remote-partial-migrate",
                      "Make sure partial migrations are handled before ops on the remote node"),
        SchedulerTest("remote-partial-migrate2",
                      "Make sure partial migration target is prefered for remote connection"),
        SchedulerTest("remote-recover-fail", "Make sure start failure causes fencing if rsc are active on remote"),
        SchedulerTest("remote-start-fail",
                      "Make sure a start failure does not result in fencing if no active resources are on remote"),
        SchedulerTest("remote-unclean2",
                      "Make monitor failure always results in fencing, even if no rsc are active on remote"),
        SchedulerTest("remote-fence-before-reconnect", "Fence before clearing recurring monitor failure"),
        SchedulerTest("remote-recovery", "Recover remote connections before attempting demotion"),
        SchedulerTest("remote-recover-connection", "Optimistically recovery of only the connection"),
        SchedulerTest("remote-recover-all", "Fencing when the connection has no home"),
        SchedulerTest("remote-recover-no-resources", "Fencing when the connection has no home and no active resources"),
        SchedulerTest("remote-recover-unknown",
                      "Fencing when the connection has no home and the remote has no operation history"),
        SchedulerTest("remote-reconnect-delay", "Waiting for remote reconnect interval to expire"),
        SchedulerTest("remote-connection-unrecoverable",
                      "Remote connection host must be fenced, with connection unrecoverable"),
        SchedulerTest("remote-connection-shutdown", "Remote connection shutdown"),
        SchedulerTest("cancel-behind-moving-remote",
                      "Route recurring monitor cancellations through original node of a moving remote connection"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("resource-discovery", "Exercises resource-discovery location constraint option"),
        SchedulerTest("rsc-discovery-per-node", "Disable resource discovery per node"),
        SchedulerTest("shutdown-lock", "Ensure shutdown lock works properly"),
        SchedulerTest("shutdown-lock-expiration", "Ensure shutdown lock expiration works properly"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("op-defaults", "Test op_defaults conditional expressions"),
        SchedulerTest("op-defaults-2", "Test op_defaults AND'ed conditional expressions"),
        SchedulerTest("op-defaults-3", "Test op_defaults precedence"),
        SchedulerTest("rsc-defaults", "Test rsc_defaults conditional expressions"),
        SchedulerTest("rsc-defaults-2", "Test rsc_defaults conditional expressions without type"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("stop-all-resources", "Test stop-all-resources=true"),
    ]),
    SchedulerTestGroup([
        SchedulerTest("ocf_degraded-remap-ocf_ok", "Test degraded remapped to OK"),
        SchedulerTest("ocf_degraded_promoted-remap-ocf_ok", "Test degraded promoted remapped to OK"),
    ]),
]

TESTS_64BIT = [
    SchedulerTestGroup([
        SchedulerTest("year-2038", "Check handling of timestamps beyond 2038-01-19 03:14:08 UTC"),
    ]),
]


def is_executable(path):
    """Check whether a file at a given path is executable."""
    try:
        return os.stat(path)[stat.ST_MODE] & stat.S_IXUSR
    except OSError:
        return False


def diff(file1, file2, **kwargs):
    """Call diff on two files."""
    return subprocess.call(["diff", "-u", "-N", "--ignore-all-space",
                            "--ignore-blank-lines", file1, file2], **kwargs)


def sort_file(filename):
    """Sort a file alphabetically."""
    with io.open(filename, "rt", encoding="utf-8") as f:
        lines = sorted(f)

    with io.open(filename, "wt", encoding="utf-8") as f:
        f.writelines(lines)


def remove_files(filenames):
    """Remove a list of files."""
    for filename in filenames:
        try:
            os.remove(filename)
        except OSError:
            pass


def normalize(filename):
    """Remove text from a file that isn't important for comparison."""
    if not hasattr(normalize, "patterns"):
        normalize.patterns = [
            re.compile(r'crm_feature_set="[^"]*"'),
            re.compile(r'batch-limit="[0-9]*"')
        ]

    if not os.path.isfile(filename):
        return

    with io.open(filename, "rt", encoding="utf-8") as f:
        lines = f.readlines()

    with io.open(filename, "wt", encoding="utf-8") as f:
        for line in lines:
            for pattern in normalize.patterns:
                line = pattern.sub("", line)

            f.write(line)


def cat(filename, dest=sys.stdout):
    """Copy a file to a destination file descriptor."""
    with io.open(filename, "rt", encoding="utf-8") as f:
        shutil.copyfileobj(f, dest)


class CtsScheduler:
    """Regression tests for Pacemaker's scheduler."""

    def _parse_args(self, argv):
        """Parse command-line arguments."""
        parser = argparse.ArgumentParser(description="Regression tests for Pacemaker's scheduler")

        parser.add_argument('-V', '--verbose', action='count',
                            help='Display any differences from expected output')
        parser.add_argument('--run', metavar='TEST',
                            help=('Run only single specified test (any further '
                                  'arguments will be passed to crm_simulate)'))
        parser.add_argument('--update', action='store_true',
                            help='Update expected results with actual results')
        parser.add_argument('-b', '--binary', metavar='PATH',
                            help='Specify path to crm_simulate')
        parser.add_argument('-i', '--io-dir', metavar='PATH',
                            help='Specify path to regression test data directory')
        parser.add_argument('-o', '--out-dir', metavar='PATH',
                            help='Specify where intermediate and output files should go')
        parser.add_argument('-v', '--valgrind', action='store_true',
                            help='Run all commands under valgrind')
        parser.add_argument('--valgrind-dhat', action='store_true',
                            help='Run all commands under valgrind with heap analyzer')
        parser.add_argument('--valgrind-skip-output', action='store_true',
                            help='If running under valgrind, do not display output')
        parser.add_argument('--testcmd-options', metavar='OPTIONS', default='',
                            help='Additional options for command under test')

        # argparse can't handle "everything after --run TEST", so grab that
        self.single_test_args = []
        narg = 0
        for arg in argv:
            narg += 1
            if arg == '--run':
                (argv, self.single_test_args) = (argv[:narg + 1], argv[narg + 1:])
                break

        self.args = parser.parse_args(argv[1:])

    def _error(self, s):
        """Print an error message."""
        print(f"      * ERROR:   {s}")

    def _failed(self, s):
        """Print a failure message."""
        print(f"      * FAILED:  {s}")

    def _get_valgrind_cmd(self):
        """Return command arguments needed (or not) to run valgrind."""
        if self.args.valgrind:
            os.environ['G_SLICE'] = "always-malloc"
            return [
                "valgrind",
                "-q",
                "--gen-suppressions=all",
                "--time-stamp=yes",
                "--trace-children=no",
                "--show-reachable=no",
                "--leak-check=full",
                "--num-callers=20",
                f"--suppressions={self.test_home}/valgrind-pcmk.suppressions"
            ]

        if self.args.valgrind_dhat:
            os.environ['G_SLICE'] = "always-malloc"
            return [
                "valgrind",
                "--tool=exp-dhat",
                "--time-stamp=yes",
                "--trace-children=no",
                "--show-top-n=100",
                "--num-callers=4"
            ]

        return []

    def _get_simulator_cmd(self):
        """Locate the simulation binary."""
        if self.args.binary is None:
            # pylint: disable=protected-access
            self.args.binary = f"{BuildOptions._BUILD_DIR}/tools/crm_simulate"

            if not is_executable(self.args.binary):
                self.args.binary = f"{BuildOptions.SBIN_DIR}/crm_simulate"

        if not is_executable(self.args.binary):
            # @TODO it would be more pythonic to raise an exception
            self._error(f"Test binary {self.args.binary} not found")
            sys.exit(ExitStatus.NOT_INSTALLED)

        return [self.args.binary] + shlex.split(self.args.testcmd_options)

    def set_schema_env(self):
        """Ensure schema directory environment variable is set, if possible."""
        try:
            return os.environ['PCMK_schema_directory']
        except KeyError:
            # pylint: disable=protected-access
            for d in [os.path.join(BuildOptions._BUILD_DIR, "xml"),
                      BuildOptions.SCHEMA_DIR]:
                if not os.path.isdir(d):
                    continue

                os.environ['PCMK_schema_directory'] = d
                return d

            return None

    def __init__(self, argv=sys.argv):
        """Create a new CtsScheduler instance."""
        # Ensure all command output is in portable locale for comparison
        os.environ['LC_ALL'] = "C"

        self._parse_args(argv)

        # Where this executable lives
        self.test_home = os.path.dirname(os.path.realpath(argv[0]))

        # Where test data resides
        if self.args.io_dir is None:
            self.args.io_dir = os.path.join(self.test_home, "scheduler")

        self.xml_input_dir = os.path.join(self.args.io_dir, "xml")
        self.expected_dir = os.path.join(self.args.io_dir, "exp")
        self.dot_expected_dir = os.path.join(self.args.io_dir, "dot")
        self.scores_dir = os.path.join(self.args.io_dir, "scores")
        self.summary_dir = os.path.join(self.args.io_dir, "summary")
        self.stderr_expected_dir = os.path.join(self.args.io_dir, "stderr")

        # Create a temporary directory to store diff file
        self.failed_dir = tempfile.mkdtemp(prefix='cts-scheduler_')

        # Where to store generated files
        if self.args.out_dir is None:
            self.args.out_dir = self.args.io_dir
            self.failed_filename = os.path.join(self.failed_dir, "test-output.diff")
        else:
            self.failed_filename = os.path.join(self.args.out_dir, "test-output.diff")

        os.environ['CIB_shadow_dir'] = self.args.out_dir

        self.failed_file = None

        self.outfile_out_dir = os.path.join(self.args.out_dir, "out")
        self.dot_out_dir = os.path.join(self.args.out_dir, "dot")
        self.scores_out_dir = os.path.join(self.args.out_dir, "scores")
        self.summary_out_dir = os.path.join(self.args.out_dir, "summary")
        self.stderr_out_dir = os.path.join(self.args.out_dir, "stderr")
        self.valgrind_out_dir = os.path.join(self.args.out_dir, "valgrind")

        # Single test mode (if requested)
        try:
            # User can give test base name or file name of a test input
            self.args.run = os.path.splitext(os.path.basename(self.args.run))[0]
        except (AttributeError, TypeError):
            pass  # --run was not specified

        self.set_schema_env()

        # Arguments needed (or not) to run commands
        self.valgrind_args = self._get_valgrind_cmd()
        self.simulate_args = self._get_simulator_cmd()

        # Test counters
        self.num_failed = 0
        self.num_tests = 0

        # Ensure that the main output directory exists
        # We don't want to create it with os.makedirs below
        if not os.path.isdir(self.args.out_dir):
            self._error("Output directory missing; can't create output files")
            sys.exit(ExitStatus.CANTCREAT)

        # Create output subdirectories if they don't exist
        try:
            os.makedirs(self.outfile_out_dir, 0o755, True)
            os.makedirs(self.dot_out_dir, 0o755, True)
            os.makedirs(self.scores_out_dir, 0o755, True)
            os.makedirs(self.summary_out_dir, 0o755, True)
            os.makedirs(self.stderr_out_dir, 0o755, True)

            if self.valgrind_args:
                os.makedirs(self.valgrind_out_dir, 0o755, True)
        except OSError as ex:
            self._error(f"Unable to create output subdirectory: {ex}")
            remove_files([
                self.outfile_out_dir,
                self.dot_out_dir,
                self.scores_out_dir,
                self.summary_out_dir,
                self.stderr_out_dir,
            ])

            sys.exit(ExitStatus.CANTCREAT)

    def _compare_files(self, filename1, filename2):
        """Add any file differences to failed results."""
        if diff(filename1, filename2, stdout=subprocess.DEVNULL) != 0:
            diff(filename1, filename2, stdout=self.failed_file, stderr=subprocess.DEVNULL)
            self.failed_file.write("\n")
            return True

        return False

    def _file_missing(self, path):
        """Return True if path does not exist or is empty."""
        return not os.path.isfile(path) or os.path.getsize(path) == 0

    def run_one(self, test_name, test_desc, test_args):
        """Run one scheduler test."""
        # pylint: disable=too-many-locals
        print(f"  Test {f'{test_name}:':41} {test_desc}")

        did_fail = False
        self.num_tests += 1

        # Test inputs
        input_filename = os.path.join(self.xml_input_dir, f"{test_name}.xml")
        expected_filename = os.path.join(self.expected_dir, f"{test_name}.exp")
        dot_expected_filename = os.path.join(self.dot_expected_dir, f"{test_name}.dot")
        scores_filename = os.path.join(self.scores_dir, f"{test_name}.scores")
        summary_filename = os.path.join(self.summary_dir, f"{test_name}.summary")
        stderr_expected_filename = os.path.join(self.stderr_expected_dir, f"{test_name}.stderr")

        # (Intermediate) test outputs
        output_filename = os.path.join(self.outfile_out_dir, f"{test_name}.out")
        dot_output_filename = os.path.join(self.dot_out_dir, f"{test_name}.dot.pe")
        score_output_filename = os.path.join(self.scores_out_dir, f"{test_name}.scores.pe")
        summary_output_filename = os.path.join(self.summary_out_dir, f"{test_name}.summary.pe")
        stderr_output_filename = os.path.join(self.stderr_out_dir, f"{test_name}.stderr.pe")
        valgrind_output_filename = os.path.join(self.valgrind_out_dir, f"{test_name}.valgrind")

        # Common arguments for running test
        test_cmd = []
        if self.valgrind_args:
            test_cmd = self.valgrind_args + [f"--log-file={valgrind_output_filename}"]

        test_cmd += self.simulate_args

        # @TODO It would be more pythonic to raise exceptions for errors,
        # then perhaps it would be nice to make a single-test class

        # Ensure necessary test inputs exist
        if not os.path.isfile(input_filename):
            self._error("No input")
            self.num_failed += 1
            return ExitStatus.NOINPUT

        if not self.args.update and not os.path.isfile(expected_filename):
            self._error("no stored output")
            return ExitStatus.NOINPUT

        # Run simulation to generate summary output
        test_cmd_full = test_cmd + ['-x', input_filename, '-S'] + test_args
        if self.args.run:  # Single test mode
            print(" ".join(test_cmd_full))

        with io.open(summary_output_filename, "wt", encoding="utf-8") as f:
            subprocess.run(test_cmd_full, stdout=f, stderr=subprocess.STDOUT,
                           env=os.environ, check=False)

        if self.args.run:
            cat(summary_output_filename)

        # Re-run simulation to generate dot, graph, and scores
        test_cmd_full = test_cmd + ['-x', input_filename,
                                    '-D', dot_output_filename,
                                    '-G', output_filename,
                                    '-sSQ'] + test_args

        with io.open(stderr_output_filename, "wt", encoding="utf-8") as f_stderr, \
             io.open(score_output_filename, "wt", encoding="utf-8") as f_score:
            rc = subprocess.call(test_cmd_full, stdout=f_score, stderr=f_stderr, env=os.environ)

        # Check for test command failure
        if rc != ExitStatus.OK:
            self._failed(f"Test returned: {rc}")
            did_fail = True
            print(" ".join(test_cmd_full))

        # Check for valgrind errors
        if self.valgrind_args and not self.args.valgrind_skip_output:
            if os.path.getsize(valgrind_output_filename) > 0:
                self._failed("Valgrind reported errors")
                did_fail = True
                cat(valgrind_output_filename)

            remove_files([valgrind_output_filename])

        # Check for core dump
        if os.path.isfile("core"):
            self._failed(f"Core-file detected: core.{test_name}")
            did_fail = True
            os.rename("core", f"{self.test_home}/core.{test_name}")

        # Check any stderr output
        if os.path.isfile(stderr_expected_filename):
            if self._compare_files(stderr_expected_filename, stderr_output_filename):
                self._failed("stderr changed")
                did_fail = True
        elif os.path.getsize(stderr_output_filename) > 0:
            self._failed("Output was written to stderr")
            did_fail = True
            cat(stderr_output_filename)

        remove_files([stderr_output_filename])

        # Check whether output graph exists, and normalize it
        if self._file_missing(output_filename):
            self._error("No graph produced")
            did_fail = True
            self.num_failed += 1
            remove_files([output_filename])
            return ExitStatus.ERROR

        normalize(output_filename)

        # Check whether dot output exists, and sort it
        if self._file_missing(dot_output_filename):
            self._error("No dot-file summary produced")
            did_fail = True
            self.num_failed += 1
            remove_files([dot_output_filename, output_filename])
            return ExitStatus.ERROR

        with io.open(dot_output_filename, "rt", encoding="utf-8") as f:
            first_line = f.readline()  # "digraph" line with opening brace
            lines = f.readlines()
            last_line = lines[-1]  # closing brace
            del lines[-1]
            lines = sorted(set(lines))  # unique sort

        with io.open(dot_output_filename, "wt", encoding="utf-8") as f:
            f.write(first_line)
            f.writelines(lines)
            f.write(last_line)

        # Check whether score output exists, and sort it
        if self._file_missing(score_output_filename):
            self._error("No allocation scores produced")
            did_fail = True
            self.num_failed += 1
            remove_files([score_output_filename, output_filename])
            return ExitStatus.ERROR

        sort_file(score_output_filename)

        if self.args.update:
            shutil.copyfile(output_filename, expected_filename)
            shutil.copyfile(dot_output_filename, dot_expected_filename)
            shutil.copyfile(score_output_filename, scores_filename)
            shutil.copyfile(summary_output_filename, summary_filename)
            print("  Updated expected outputs")

        if self._compare_files(summary_filename, summary_output_filename):
            self._failed("summary changed")
            did_fail = True

        if self._compare_files(dot_expected_filename, dot_output_filename):
            self._failed("dot-file summary changed")
            did_fail = True
        else:
            remove_files([dot_output_filename])

        if self._compare_files(expected_filename, output_filename):
            self._failed("xml-file changed")
            did_fail = True

        if self._compare_files(scores_filename, score_output_filename):
            self._failed("scores-file changed")
            did_fail = True

        remove_files([output_filename,
                      dot_output_filename,
                      score_output_filename,
                      summary_output_filename])

        if did_fail:
            self.num_failed += 1
            return ExitStatus.ERROR

        return ExitStatus.OK

    def run_all(self):
        """Run all defined tests."""
        if platform.architecture()[0] == "64bit":
            TESTS.extend(TESTS_64BIT)

        for group in TESTS:
            for test in group.tests:
                self.run_one(test.name, test.desc, test.args)

            print()

    def _print_summary(self):
        """Print a summary of parameters for this test run."""
        print(f"Test home is:\t{self.test_home}")
        print(f"Test binary is:\t{self.args.binary}")

        if 'PCMK_schema_directory' in os.environ:
            print(f"Schema home is:\t{os.environ['PCMK_schema_directory']}")

        if self.valgrind_args:
            print("Activating memory testing with valgrind")

        print()

    def _test_results(self):
        """Report test results."""
        if self.num_failed == 0:
            shutil.rmtree(self.failed_dir)
            return ExitStatus.OK

        if self._file_missing(self.failed_filename):
            self._error(f"{self.num_failed} (of {self.num_tests}) tests failed (no diff results)")
            if os.path.isfile(self.failed_filename):
                shutil.rmtree(self.failed_dir)
        elif self.args.verbose:
            self._error(f"Results of {self.num_failed} failed tests (out of {self.num_tests}):")
            cat(self.failed_filename)
        else:
            self._error(f"Results of {self.num_failed} failed tests (out of {self.num_tests}) "
                        f"are in {self.failed_filename}")
            self._error("Use -V to display them after running the tests")

        return ExitStatus.ERROR

    def find_test(self, name):
        """Return the SchedulerTest object with the given name."""
        if platform.architecture()[0] == "64bit":
            TESTS.extend(TESTS_64BIT)

        for group in TESTS:
            for test in group.tests:
                if test.name == name:
                    return test

        return None

    def run(self):
        """Run test(s) as specified."""
        # Check for pre-existing core so we don't think it's from us
        if os.path.exists("core"):
            self._failed(f"Can't run with core already present in {self.test_home}")
            return ExitStatus.OSFILE

        self._print_summary()

        # Zero out the error log
        # pylint: disable=consider-using-with
        self.failed_file = io.open(self.failed_filename, "wt", encoding="utf-8")

        if self.args.run is None:
            print(f"Performing the following tests from {self.args.io_dir}")
            print()
            self.run_all()
            print()
            self.failed_file.close()
            rc = self._test_results()
        else:
            # Find the test we were asked to run
            test = self.find_test(self.args.run)

            if test is None:
                print(f"No test named {self.args.run}")
                return ExitStatus.INVALID_PARAM

            # If no arguments were given on the command line, default to the ones
            # contained in the test
            if self.single_test_args:
                args = self.single_test_args
            else:
                args = test.args

            rc = self.run_one(test.name, test.desc, args)
            self.failed_file.close()

            if self.num_failed > 0:
                print(f"\nFailures:\nThese have also been written to: {self.failed_filename}\n")
                cat(self.failed_filename)
                shutil.rmtree(self.failed_dir)

        return rc


if __name__ == "__main__":
    sys.exit(CtsScheduler().run())

# vim: set filetype=python expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=120:
