root/lib/pengine/bundle.c

/* */

DEFINITIONS

1. pe__bundle_max
2. pe__bundle_max_per_node
3. next_ip
4. allocate_ip
5. create_resource
6. valid_network
7. create_ip_resource
8. container_agent_str
9. create_container_resource
10. disallow_node
11. create_remote_resource
12. create_replica_resources
13. mount_add
14. mount_free
15. port_free
16. replica_for_remote
17. pe__bundle_needs_remote_name
18. pe__add_bundle_remote_name
19. pe__unpack_bundle
20. replica_resource_active
21. pe__bundle_active
22. pe__find_bundle_replica
23. print_rsc_in_list
24. bundle_print_xml
25. PCMK__OUTPUT_ARGS
26. pe__bundle_replica_output_html
27. get_unmanaged_str
28. PCMK__OUTPUT_ARGS
29. pe__bundle_replica_output_text
30. PCMK__OUTPUT_ARGS
31. print_bundle_replica
32. pe__print_bundle
33. free_bundle_replica
34. pe__free_bundle
35. pe__bundle_resource_state
36. pe_bundle_replicas
37. pe__count_bundle
38. pe__bundle_is_filtered
39. pe__bundle_containers
40. pe__bundle_active_node

   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdint.h>
  14 
  15 #include <crm/pengine/rules.h>
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/msg_xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 #define PE__VARIANT_BUNDLE 1
  24 #include "./variant.h"
  25 
  26 /*!
  27  * \internal
  28  * \brief Get maximum number of bundle replicas allowed to run
  29  *
  30  * \param[in] rsc  Bundle or bundled resource to check
  31  *
  32  * \return Maximum replicas for bundle corresponding to \p rsc
  33  */
  34 int
  35 pe__bundle_max(const pe_resource_t *rsc)
     /*  */
  36 {
  37     const pe__bundle_variant_data_t *bundle_data = NULL;
  38 
  39     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
  40     return bundle_data->nreplicas;
  41 }
  42 
  43 /*!
  44  * \internal
  45  * \brief Get maximum number of bundle replicas allowed to run on one node
  46  *
  47  * \param[in] rsc  Bundle or bundled resource to check
  48  *
  49  * \return Maximum replicas per node for bundle corresponding to \p rsc
  50  */
  51 int
  52 pe__bundle_max_per_node(const pe_resource_t *rsc)
     /*  */
  53 {
  54     const pe__bundle_variant_data_t *bundle_data = NULL;
  55 
  56     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
  57     return bundle_data->nreplicas_per_host;
  58 }
  59 
  60 static char *
  61 next_ip(const char *last_ip)
     /*  */
  62 {
  63     unsigned int oct1 = 0;
  64     unsigned int oct2 = 0;
  65     unsigned int oct3 = 0;
  66     unsigned int oct4 = 0;
  67     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
  68 
  69     if (rc != 4) {
  70         /*@ TODO check for IPv6 */
  71         return NULL;
  72 
  73     } else if (oct3 > 253) {
  74         return NULL;
  75 
  76     } else if (oct4 > 253) {
  77         ++oct3;
  78         oct4 = 1;
  79 
  80     } else {
  81         ++oct4;
  82     }
  83 
  84     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
  85 }
  86 
  87 static void
  88 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
  89             GString *buffer)
  90 {
  91     if(data->ip_range_start == NULL) {
  92         return;
  93 
  94     } else if(data->ip_last) {
  95         replica->ipaddr = next_ip(data->ip_last);
  96 
  97     } else {
  98         replica->ipaddr = strdup(data->ip_range_start);
  99     }
 100 
 101     data->ip_last = replica->ipaddr;
 102     switch (data->agent_type) {
 103         case PE__CONTAINER_AGENT_DOCKER:
 104         case PE__CONTAINER_AGENT_PODMAN:
 105             if (data->add_host) {
 106                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
 107                                        data->prefix, replica->offset,
 108                                        replica->ipaddr);
 109             } else {
 110                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 111                                        replica->ipaddr, data->prefix,
 112                                        replica->offset);
 113             }
 114             break;
 115 
 116         case PE__CONTAINER_AGENT_RKT:
 117             g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 118                                    replica->ipaddr, data->prefix,
 119                                    replica->offset);
 120             break;
 121 
 122         default: // PE__CONTAINER_AGENT_UNKNOWN
 123             break;
 124     }
 125 }
 126 
 127 static xmlNode *
 128 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
 129 {
 130     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 131 
 132     crm_xml_add(rsc, XML_ATTR_ID, name);
 133     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
 134     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
 135     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
 136 
 137     return rsc;
 138 }
 139 
 140 /*!
 141  * \internal
 142  * \brief Check whether cluster can manage resource inside container
 143  *
 144  * \param[in,out] data  Container variant data
 145  *
 146  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 147  *
 148  * \note The resource is manageable if an IP range or control port has been
 149  *       specified. If a control port is used without an IP range, replicas per
 150  *       host must be 1.
 151  */
 152 static bool
 153 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 154 {
 155     if(data->ip_range_start) {
 156         return TRUE;
 157     }
 158     if(data->control_port) {
 159         if(data->nreplicas_per_host > 1) {
 160             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 161             data->nreplicas_per_host = 1;
 162             // @TODO to be sure: pe__clear_resource_flags(rsc, pe_rsc_unique);
 163         }
 164         return TRUE;
 165     }
 166     return FALSE;
 167 }
 168 
 169 static int
 170 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 171                    pe__bundle_replica_t *replica)
 172 {
 173     if(data->ip_range_start) {
 174         char *id = NULL;
 175         xmlNode *xml_ip = NULL;
 176         xmlNode *xml_obj = NULL;
 177 
 178         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 179         crm_xml_sanitize_id(id);
 180         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 181         free(id);
 182 
 183         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 184         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 185                        data->prefix, replica->offset);
 186 
 187         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 188         if(data->host_network) {
 189             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 190         }
 191 
 192         if(data->host_netmask) {
 193             crm_create_nvpair_xml(xml_obj, NULL,
 194                                   "cidr_netmask", data->host_netmask);
 195 
 196         } else {
 197             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 198         }
 199 
 200         xml_obj = create_xml_node(xml_ip, "operations");
 201         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 202 
 203         // TODO: Other ops? Timeouts and intervals from underlying resource?
 204 
 205         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
 206                                 parent->cluster) != pcmk_rc_ok) {
 207             return pcmk_rc_unpack_error;
 208         }
 209 
 210         parent->children = g_list_append(parent->children, replica->ip);
 211     }
 212     return pcmk_rc_ok;
 213 }
 214 
 215 static const char*
 216 container_agent_str(enum pe__container_agent t)
     /*  */
 217 {
 218     switch (t) {
 219         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
 220         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
 221         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
 222         default: // PE__CONTAINER_AGENT_UNKNOWN
 223             break;
 224     }
 225     return PE__CONTAINER_AGENT_UNKNOWN_S;
 226 }
 227 
 228 static int
 229 create_container_resource(pe_resource_t *parent,
     /*  */
 230                           const pe__bundle_variant_data_t *data,
 231                           pe__bundle_replica_t *replica)
 232 {
 233     char *id = NULL;
 234     xmlNode *xml_container = NULL;
 235     xmlNode *xml_obj = NULL;
 236 
 237     // Agent-specific
 238     const char *hostname_opt = NULL;
 239     const char *env_opt = NULL;
 240     const char *agent_str = NULL;
 241     int volid = 0;  // rkt-only
 242 
 243     GString *buffer = NULL;
 244     GString *dbuffer = NULL;
 245 
 246     // Where syntax differences are drop-in replacements, set them now
 247     switch (data->agent_type) {
 248         case PE__CONTAINER_AGENT_DOCKER:
 249         case PE__CONTAINER_AGENT_PODMAN:
 250             hostname_opt = "-h ";
 251             env_opt = "-e ";
 252             break;
 253         case PE__CONTAINER_AGENT_RKT:
 254             hostname_opt = "--hostname=";
 255             env_opt = "--environment=";
 256             break;
 257         default:    // PE__CONTAINER_AGENT_UNKNOWN
 258             return pcmk_rc_unpack_error;
 259     }
 260     agent_str = container_agent_str(data->agent_type);
 261 
 262     buffer = g_string_sized_new(4096);
 263 
 264     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
 265                            replica->offset);
 266     crm_xml_sanitize_id(id);
 267     xml_container = create_resource(id, "heartbeat", agent_str);
 268     free(id);
 269 
 270     xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 271     crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 272 
 273     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 274     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 275     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 276     crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 277 
 278     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
 279         g_string_append(buffer, " --restart=no");
 280     }
 281 
 282     /* Set a container hostname only if we have an IP to map it to. The user can
 283      * set -h or --uts=host themselves if they want a nicer name for logs, but
 284      * this makes applications happy who need their  hostname to match the IP
 285      * they bind to.
 286      */
 287     if (data->ip_range_start != NULL) {
 288         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
 289                                replica->offset);
 290     }
 291     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 292 
 293     if (data->container_network != NULL) {
 294         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
 295     }
 296 
 297     if (data->control_port != NULL) {
 298         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_remote_port=",
 299                       data->control_port, NULL);
 300     } else {
 301         g_string_append_printf(buffer, " %sPCMK_remote_port=%d", env_opt,
 302                                DEFAULT_REMOTE_PORT);
 303     }
 304 
 305     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
 306         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
 307         char *source = NULL;
 308 
 309         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 310             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
 311                                        replica->offset);
 312             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
 313         }
 314 
 315         switch (data->agent_type) {
 316             case PE__CONTAINER_AGENT_DOCKER:
 317             case PE__CONTAINER_AGENT_PODMAN:
 318                 pcmk__g_strcat(buffer,
 319                                " -v ", pcmk__s(source, mount->source),
 320                                ":", mount->target, NULL);
 321 
 322                 if (mount->options != NULL) {
 323                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
 324                 }
 325                 break;
 326             case PE__CONTAINER_AGENT_RKT:
 327                 g_string_append_printf(buffer,
 328                                        " --volume vol%d,kind=host,"
 329                                        "source=%s%s%s "
 330                                        "--mount volume=vol%d,target=%s",
 331                                        volid, pcmk__s(source, mount->source),
 332                                        (mount->options != NULL)? "," : "",
 333                                        pcmk__s(mount->options, ""),
 334                                        volid, mount->target);
 335                 volid++;
 336                 break;
 337             default:
 338                 break;
 339         }
 340         free(source);
 341     }
 342 
 343     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
 344         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 345 
 346         switch (data->agent_type) {
 347             case PE__CONTAINER_AGENT_DOCKER:
 348             case PE__CONTAINER_AGENT_PODMAN:
 349                 if (replica->ipaddr != NULL) {
 350                     pcmk__g_strcat(buffer,
 351                                    " -p ", replica->ipaddr, ":", port->source,
 352                                    ":", port->target, NULL);
 353 
 354                 } else if (!pcmk__str_eq(data->container_network, "host",
 355                                          pcmk__str_none)) {
 356                     // No need to do port mapping if net == host
 357                     pcmk__g_strcat(buffer,
 358                                    " -p ", port->source, ":", port->target,
 359                                    NULL);
 360                 }
 361                 break;
 362             case PE__CONTAINER_AGENT_RKT:
 363                 if (replica->ipaddr != NULL) {
 364                     pcmk__g_strcat(buffer,
 365                                    " --port=", port->target,
 366                                    ":", replica->ipaddr, ":", port->source,
 367                                    NULL);
 368                 } else {
 369                     pcmk__g_strcat(buffer,
 370                                    " --port=", port->target, ":", port->source,
 371                                    NULL);
 372                 }
 373                 break;
 374             default:
 375                 break;
 376         }
 377     }
 378 
 379     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
 380      * it would cause restarts during rolling upgrades.
 381      *
 382      * In a previous version of the container resource creation logic, if
 383      * data->launcher_options is not NULL, we append
 384      * (" %s", data->launcher_options) even if data->launcher_options is an
 385      * empty string. Likewise for data->container_host_options. Using
 386      *
 387      *     pcmk__add_word(buffer, 0, data->launcher_options)
 388      *
 389      * removes that extra trailing space, causing a resource definition change.
 390      */
 391     if (data->launcher_options != NULL) {
 392         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
 393     }
 394 
 395     if (data->container_host_options != NULL) {
 396         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
 397     }
 398 
 399     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
 400                           (const char *) buffer->str);
 401     g_string_free(buffer, TRUE);
 402 
 403     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
 404                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
 405     if (dbuffer != NULL) {
 406         g_string_free(dbuffer, TRUE);
 407     }
 408 
 409     if (replica->child != NULL) {
 410         if (data->container_command != NULL) {
 411             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 412                                   data->container_command);
 413         } else {
 414             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 415                                   SBIN_DIR "/pacemaker-remoted");
 416         }
 417 
 418         /* TODO: Allow users to specify their own?
 419          *
 420          * We just want to know if the container is alive; we'll monitor the
 421          * child independently.
 422          */
 423         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 424 #if 0
 425         /* @TODO Consider supporting the use case where we can start and stop
 426          * resources, but not proxy local commands (such as setting node
 427          * attributes), by running the local executor in stand-alone mode.
 428          * However, this would probably be better done via ACLs as with other
 429          * Pacemaker Remote nodes.
 430          */
 431     } else if ((child != NULL) && data->untrusted) {
 432         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 433                               CRM_DAEMON_DIR "/pacemaker-execd");
 434         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 435                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 436 #endif
 437     } else {
 438         if (data->container_command != NULL) {
 439             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 440                                   data->container_command);
 441         }
 442 
 443         /* TODO: Allow users to specify their own?
 444          *
 445          * We don't know what's in the container, so we just want to know if it
 446          * is alive.
 447          */
 448         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 449     }
 450 
 451     xml_obj = create_xml_node(xml_container, "operations");
 452     crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 453 
 454     // TODO: Other ops? Timeouts and intervals from underlying resource?
 455     if (pe__unpack_resource(xml_container, &replica->container, parent,
 456                             parent->cluster) != pcmk_rc_ok) {
 457         return pcmk_rc_unpack_error;
 458     }
 459     pe__set_resource_flags(replica->container, pe_rsc_replica_container);
 460     parent->children = g_list_append(parent->children, replica->container);
 461 
 462     return pcmk_rc_ok;
 463 }
 464 
 465 /*!
 466  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 467  *
 468  * \param[in,out] rsc    Resource to modify
 469  * \param[in]     uname  Name of node to ban
 470  */
 471 static void
 472 disallow_node(pe_resource_t *rsc, const char *uname)
     /*  */
 473 {
 474     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 475 
 476     if (match) {
 477         ((pe_node_t *) match)->weight = -INFINITY;
 478         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
 479     }
 480     if (rsc->children) {
 481         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
 482     }
 483 }
 484 
 485 static int
 486 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 487                        pe__bundle_replica_t *replica)
 488 {
 489     if (replica->child && valid_network(data)) {
 490         GHashTableIter gIter;
 491         pe_node_t *node = NULL;
 492         xmlNode *xml_remote = NULL;
 493         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 494         char *port_s = NULL;
 495         const char *uname = NULL;
 496         const char *connect_name = NULL;
 497 
 498         if (pe_find_resource(parent->cluster->resources, id) != NULL) {
 499             free(id);
 500             // The biggest hammer we have
 501             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 502                                    replica->child->id, replica->offset);
 503             //@TODO return error instead of asserting?
 504             CRM_ASSERT(pe_find_resource(parent->cluster->resources,
 505                                         id) == NULL);
 506         }
 507 
 508         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 509          * connection does not have its own IP is a magic string that we use to
 510          * support nested remotes (i.e. a bundle running on a remote node).
 511          */
 512         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 513 
 514         if (data->control_port == NULL) {
 515             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 516         }
 517 
 518         /* This sets replica->container as replica->remote's container, which is
 519          * similar to what happens with guest nodes. This is how the scheduler
 520          * knows that the bundle node is fenced by recovering the container, and
 521          * that remote should be ordered relative to the container.
 522          */
 523         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 524                                           NULL, NULL, NULL,
 525                                           connect_name, (data->control_port?
 526                                           data->control_port : port_s));
 527         free(port_s);
 528 
 529         /* Abandon our created ID, and pull the copy from the XML, because we
 530          * need something that will get freed during data set cleanup to use as
 531          * the node ID and uname.
 532          */
 533         free(id);
 534         id = NULL;
 535         uname = ID(xml_remote);
 536 
 537         /* Ensure a node has been created for the guest (it may have already
 538          * been, if it has a permanent node attribute), and ensure its weight is
 539          * -INFINITY so no other resources can run on it.
 540          */
 541         node = pe_find_node(parent->cluster->nodes, uname);
 542         if (node == NULL) {
 543             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 544                                   parent->cluster);
 545         } else {
 546             node->weight = -INFINITY;
 547         }
 548         node->rsc_discover_mode = pe_discover_never;
 549 
 550         /* unpack_remote_nodes() ensures that each remote node and guest node
 551          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
 552          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
 553          * what nodes will be needed, so we do it just above.
 554          *
 555          * Worse, that means that the node may have been utilized while
 556          * unpacking other resources, without our weight correction. The most
 557          * likely place for this to happen is when pe__unpack_resource() calls
 558          * resource_location() to set a default score in symmetric clusters.
 559          * This adds a node *copy* to each resource's allowed nodes, and these
 560          * copies will have the wrong weight.
 561          *
 562          * As a hacky workaround, fix those copies here.
 563          *
 564          * @TODO Possible alternative: ensure bundles are unpacked before other
 565          * resources, so the weight is correct before any copies are made.
 566          */
 567         g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
 568                        (gpointer) uname);
 569 
 570         replica->node = pe__copy_node(node);
 571         replica->node->weight = 500;
 572         replica->node->rsc_discover_mode = pe_discover_exclusive;
 573 
 574         /* Ensure the node shows up as allowed and with the correct discovery set */
 575         if (replica->child->allowed_nodes != NULL) {
 576             g_hash_table_destroy(replica->child->allowed_nodes);
 577         }
 578         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 579         g_hash_table_insert(replica->child->allowed_nodes,
 580                             (gpointer) replica->node->details->id,
 581                             pe__copy_node(replica->node));
 582 
 583         {
 584             pe_node_t *copy = pe__copy_node(replica->node);
 585             copy->weight = -INFINITY;
 586             g_hash_table_insert(replica->child->parent->allowed_nodes,
 587                                 (gpointer) replica->node->details->id, copy);
 588         }
 589         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
 590                                 parent->cluster) != pcmk_rc_ok) {
 591             return pcmk_rc_unpack_error;
 592         }
 593 
 594         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 595         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 596             if (pe__is_guest_or_remote_node(node)) {
 597                 /* Remote resources can only run on 'normal' cluster node */
 598                 node->weight = -INFINITY;
 599             }
 600         }
 601 
 602         replica->node->details->remote_rsc = replica->remote;
 603 
 604         // Ensure pe__is_guest_node() functions correctly immediately
 605         replica->remote->container = replica->container;
 606 
 607         /* A bundle's #kind is closer to "container" (guest node) than the
 608          * "remote" set by pe_create_node().
 609          */
 610         g_hash_table_insert(replica->node->details->attrs,
 611                             strdup(CRM_ATTR_KIND), strdup("container"));
 612 
 613         /* One effect of this is that setup_container() will add
 614          * replica->remote to replica->container's fillers, which will make
 615          * pe__resource_contains_guest_node() true for replica->container.
 616          *
 617          * replica->child does NOT get added to replica->container's fillers.
 618          * The only noticeable effect if it did would be for its fail count to
 619          * be taken into account when checking replica->container's migration
 620          * threshold.
 621          */
 622         parent->children = g_list_append(parent->children, replica->remote);
 623     }
 624     return pcmk_rc_ok;
 625 }
 626 
 627 static int
 628 create_replica_resources(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 629                          pe__bundle_replica_t *replica)
 630 {
 631     int rc = pcmk_rc_ok;
 632 
 633     rc = create_container_resource(parent, data, replica);
 634     if (rc != pcmk_rc_ok) {
 635         return rc;
 636     }
 637 
 638     rc = create_ip_resource(parent, data, replica);
 639     if (rc != pcmk_rc_ok) {
 640         return rc;
 641     }
 642 
 643     rc = create_remote_resource(parent, data, replica);
 644     if (rc != pcmk_rc_ok) {
 645         return rc;
 646     }
 647 
 648     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
 649         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 650     }
 651 
 652     if (replica->remote != NULL) {
 653         /*
 654          * Allow the remote connection resource to be allocated to a
 655          * different node than the one on which the container is active.
 656          *
 657          * This makes it possible to have Pacemaker Remote nodes running
 658          * containers with pacemaker-remoted inside in order to start
 659          * services inside those containers.
 660          */
 661         pe__set_resource_flags(replica->remote, pe_rsc_allow_remote_remotes);
 662     }
 663     return rc;
 664 }
 665 
 666 static void
 667 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 668           const char *target, const char *options, uint32_t flags)
 669 {
 670     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 671 
 672     CRM_ASSERT(mount != NULL);
 673     mount->source = strdup(source);
 674     mount->target = strdup(target);
 675     pcmk__str_update(&mount->options, options);
 676     mount->flags = flags;
 677     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 678 }
 679 
 680 static void
 681 mount_free(pe__bundle_mount_t *mount)
     /*  */
 682 {
 683     free(mount->source);
 684     free(mount->target);
 685     free(mount->options);
 686     free(mount);
 687 }
 688 
 689 static void
 690 port_free(pe__bundle_port_t *port)
     /*  */
 691 {
 692     free(port->source);
 693     free(port->target);
 694     free(port);
 695 }
 696 
 697 static pe__bundle_replica_t *
 698 replica_for_remote(pe_resource_t *remote)
     /*  */
 699 {
 700     pe_resource_t *top = remote;
 701     pe__bundle_variant_data_t *bundle_data = NULL;
 702 
 703     if (top == NULL) {
 704         return NULL;
 705     }
 706 
 707     while (top->parent != NULL) {
 708         top = top->parent;
 709     }
 710 
 711     get_bundle_variant_data(bundle_data, top);
 712     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 713          gIter = gIter->next) {
 714         pe__bundle_replica_t *replica = gIter->data;
 715 
 716         if (replica->remote == remote) {
 717             return replica;
 718         }
 719     }
 720     CRM_LOG_ASSERT(FALSE);
 721     return NULL;
 722 }
 723 
 724 bool
 725 pe__bundle_needs_remote_name(pe_resource_t *rsc)
     /*  */
 726 {
 727     const char *value;
 728     GHashTable *params = NULL;
 729 
 730     if (rsc == NULL) {
 731         return false;
 732     }
 733 
 734     // Use NULL node since pcmk__bundle_expand() uses that to set value
 735     params = pe_rsc_params(rsc, NULL, rsc->cluster);
 736     value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 737 
 738     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 739            && xml_contains_remote_node(rsc->xml);
 740 }
 741 
 742 const char *
 743 pe__add_bundle_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set,
     /*  */
 744                            xmlNode *xml, const char *field)
 745 {
 746     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 747 
 748     pe_node_t *node = NULL;
 749     pe__bundle_replica_t *replica = NULL;
 750 
 751     if (!pe__bundle_needs_remote_name(rsc)) {
 752         return NULL;
 753     }
 754 
 755     replica = replica_for_remote(rsc);
 756     if (replica == NULL) {
 757         return NULL;
 758     }
 759 
 760     node = replica->container->allocated_to;
 761     if (node == NULL) {
 762         /* If it won't be running anywhere after the
 763          * transition, go with where it's running now.
 764          */
 765         node = pe__current_node(replica->container);
 766     }
 767 
 768     if(node == NULL) {
 769         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 770         return NULL;
 771     }
 772 
 773     crm_trace("Setting address for bundle connection %s to bundle host %s",
 774               rsc->id, pe__node_name(node));
 775     if(xml != NULL && field != NULL) {
 776         crm_xml_add(xml, field, node->details->uname);
 777     }
 778 
 779     return node->details->uname;
 780 }
 781 
 782 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
 783         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
 784                                    "Bundle mount", ID(mount_xml), flags,    \
 785                                    (flags_to_set), #flags_to_set);          \
 786     } while (0)
 787 
 788 gboolean
 789 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
 790 {
 791     const char *value = NULL;
 792     xmlNode *xml_obj = NULL;
 793     xmlNode *xml_resource = NULL;
 794     pe__bundle_variant_data_t *bundle_data = NULL;
 795     bool need_log_mount = TRUE;
 796 
 797     CRM_ASSERT(rsc != NULL);
 798     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
 799 
 800     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
 801     rsc->variant_opaque = bundle_data;
 802     bundle_data->prefix = strdup(rsc->id);
 803 
 804     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
 805     if (xml_obj != NULL) {
 806         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
 807     } else {
 808         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
 809         if (xml_obj != NULL) {
 810             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
 811         } else {
 812             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
 813             if (xml_obj != NULL) {
 814                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
 815             } else {
 816                 return FALSE;
 817             }
 818         }
 819     }
 820 
 821     // Use 0 for default, minimum, and invalid promoted-max
 822     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
 823     if (value == NULL) {
 824         // @COMPAT deprecated since 2.0.0
 825         value = crm_element_value(xml_obj, "masters");
 826     }
 827     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
 828 
 829     // Default replicas to promoted-max if it was specified and 1 otherwise
 830     value = crm_element_value(xml_obj, "replicas");
 831     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
 832         bundle_data->nreplicas = bundle_data->promoted_max;
 833     } else {
 834         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
 835     }
 836 
 837     /*
 838      * Communication between containers on the same host via the
 839      * floating IPs only works if the container is started with:
 840      *   --userland-proxy=false --ip-masq=false
 841      */
 842     value = crm_element_value(xml_obj, "replicas-per-host");
 843     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
 844     if (bundle_data->nreplicas_per_host == 1) {
 845         pe__clear_resource_flags(rsc, pe_rsc_unique);
 846     }
 847 
 848     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
 849     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
 850     bundle_data->image = crm_element_value_copy(xml_obj, "image");
 851     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
 852 
 853     xml_obj = first_named_child(rsc->xml, "network");
 854     if(xml_obj) {
 855 
 856         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
 857         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
 858         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
 859         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
 860         value = crm_element_value(xml_obj, "add-host");
 861         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
 862             bundle_data->add_host = TRUE;
 863         }
 864 
 865         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
 866              xml_child = pcmk__xe_next(xml_child)) {
 867 
 868             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
 869             port->source = crm_element_value_copy(xml_child, "port");
 870 
 871             if(port->source == NULL) {
 872                 port->source = crm_element_value_copy(xml_child, "range");
 873             } else {
 874                 port->target = crm_element_value_copy(xml_child, "internal-port");
 875             }
 876 
 877             if(port->source != NULL && strlen(port->source) > 0) {
 878                 if(port->target == NULL) {
 879                     port->target = strdup(port->source);
 880                 }
 881                 bundle_data->ports = g_list_append(bundle_data->ports, port);
 882 
 883             } else {
 884                 pe_err("Invalid port directive %s", ID(xml_child));
 885                 port_free(port);
 886             }
 887         }
 888     }
 889 
 890     xml_obj = first_named_child(rsc->xml, "storage");
 891     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
 892          xml_child = pcmk__xe_next(xml_child)) {
 893 
 894         const char *source = crm_element_value(xml_child, "source-dir");
 895         const char *target = crm_element_value(xml_child, "target-dir");
 896         const char *options = crm_element_value(xml_child, "options");
 897         int flags = pe__bundle_mount_none;
 898 
 899         if (source == NULL) {
 900             source = crm_element_value(xml_child, "source-dir-root");
 901             pe__set_bundle_mount_flags(xml_child, flags,
 902                                        pe__bundle_mount_subdir);
 903         }
 904 
 905         if (source && target) {
 906             mount_add(bundle_data, source, target, options, flags);
 907             if (strcmp(target, "/var/log") == 0) {
 908                 need_log_mount = FALSE;
 909             }
 910         } else {
 911             pe_err("Invalid mount directive %s", ID(xml_child));
 912         }
 913     }
 914 
 915     xml_obj = first_named_child(rsc->xml, "primitive");
 916     if (xml_obj && valid_network(bundle_data)) {
 917         char *value = NULL;
 918         xmlNode *xml_set = NULL;
 919 
 920         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
 921 
 922         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
 923          * part of the resource name, so that bundles don't restart in a rolling
 924          * upgrade. (It also avoids needing to change regression tests.)
 925          */
 926         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
 927                       (bundle_data->promoted_max? "master"
 928                       : (const char *)xml_resource->name));
 929 
 930         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
 931         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
 932 
 933         crm_create_nvpair_xml(xml_set, NULL,
 934                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
 935 
 936         value = pcmk__itoa(bundle_data->nreplicas);
 937         crm_create_nvpair_xml(xml_set, NULL,
 938                               XML_RSC_ATTR_INCARNATION_MAX, value);
 939         free(value);
 940 
 941         value = pcmk__itoa(bundle_data->nreplicas_per_host);
 942         crm_create_nvpair_xml(xml_set, NULL,
 943                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
 944         free(value);
 945 
 946         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
 947                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
 948 
 949         if (bundle_data->promoted_max) {
 950             crm_create_nvpair_xml(xml_set, NULL,
 951                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
 952 
 953             value = pcmk__itoa(bundle_data->promoted_max);
 954             crm_create_nvpair_xml(xml_set, NULL,
 955                                   XML_RSC_ATTR_PROMOTED_MAX, value);
 956             free(value);
 957         }
 958 
 959         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
 960         add_node_copy(xml_resource, xml_obj);
 961 
 962     } else if(xml_obj) {
 963         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
 964                rsc->id, ID(xml_obj));
 965         return FALSE;
 966     }
 967 
 968     if(xml_resource) {
 969         int lpc = 0;
 970         GList *childIter = NULL;
 971         pe__bundle_port_t *port = NULL;
 972         GString *buffer = NULL;
 973 
 974         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
 975                                 data_set) != pcmk_rc_ok) {
 976             return FALSE;
 977         }
 978 
 979         /* Currently, we always map the default authentication key location
 980          * into the same location inside the container.
 981          *
 982          * Ideally, we would respect the host's PCMK_authkey_location, but:
 983          * - it may be different on different nodes;
 984          * - the actual connection will do extra checking to make sure the key
 985          *   file exists and is readable, that we can't do here on the DC
 986          * - tools such as crm_resource and crm_simulate may not have the same
 987          *   environment variables as the cluster, causing operation digests to
 988          *   differ
 989          *
 990          * Always using the default location inside the container is fine,
 991          * because we control the pacemaker_remote environment, and it avoids
 992          * having to pass another environment variable to the container.
 993          *
 994          * @TODO A better solution may be to have only pacemaker_remote use the
 995          * environment variable, and have the cluster nodes use a new
 996          * cluster option for key location. This would introduce the limitation
 997          * of the location being the same on all cluster nodes, but that's
 998          * reasonable.
 999          */
1000         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1001                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1002 
1003         if (need_log_mount) {
1004             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1005                       pe__bundle_mount_subdir);
1006         }
1007 
1008         port = calloc(1, sizeof(pe__bundle_port_t));
1009         if(bundle_data->control_port) {
1010             port->source = strdup(bundle_data->control_port);
1011         } else {
1012             /* If we wanted to respect PCMK_remote_port, we could use
1013              * crm_default_remote_port() here and elsewhere in this file instead
1014              * of DEFAULT_REMOTE_PORT.
1015              *
1016              * However, it gains nothing, since we control both the container
1017              * environment and the connection resource parameters, and the user
1018              * can use a different port if desired by setting control-port.
1019              */
1020             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1021         }
1022         port->target = strdup(port->source);
1023         bundle_data->ports = g_list_append(bundle_data->ports, port);
1024 
1025         buffer = g_string_sized_new(1024);
1026         for (childIter = bundle_data->child->children; childIter != NULL;
1027              childIter = childIter->next) {
1028 
1029             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1030 
1031             replica->child = childIter->data;
1032             replica->child->exclusive_discover = TRUE;
1033             replica->offset = lpc++;
1034 
1035             // Ensure the child's notify gets set based on the underlying primitive's value
1036             if (pcmk_is_set(replica->child->flags, pe_rsc_notify)) {
1037                 pe__set_resource_flags(bundle_data->child, pe_rsc_notify);
1038             }
1039 
1040             allocate_ip(bundle_data, replica, buffer);
1041             bundle_data->replicas = g_list_append(bundle_data->replicas,
1042                                                   replica);
1043             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1044                                                                 XML_RSC_ATTR_TARGET);
1045         }
1046         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1047 
1048         if (bundle_data->attribute_target) {
1049             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1050                                  strdup(bundle_data->attribute_target));
1051             g_hash_table_replace(bundle_data->child->meta,
1052                                  strdup(XML_RSC_ATTR_TARGET),
1053                                  strdup(bundle_data->attribute_target));
1054         }
1055 
1056     } else {
1057         // Just a naked container, no pacemaker-remote
1058         GString *buffer = g_string_sized_new(1024);
1059 
1060         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1061             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1062 
1063             replica->offset = lpc;
1064             allocate_ip(bundle_data, replica, buffer);
1065             bundle_data->replicas = g_list_append(bundle_data->replicas,
1066                                                   replica);
1067         }
1068         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1069     }
1070 
1071     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1072          gIter = gIter->next) {
1073         pe__bundle_replica_t *replica = gIter->data;
1074 
1075         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
1076             pe_err("Failed unpacking resource %s", rsc->id);
1077             rsc->fns->free(rsc);
1078             return FALSE;
1079         }
1080 
1081         /* Utilization needs special handling for bundles. It makes no sense for
1082          * the inner primitive to have utilization, because it is tied
1083          * one-to-one to the guest node created by the container resource -- and
1084          * there's no way to set capacities for that guest node anyway.
1085          *
1086          * What the user really wants is to configure utilization for the
1087          * container. However, the schema only allows utilization for
1088          * primitives, and the container resource is implicit anyway, so the
1089          * user can *only* configure utilization for the inner primitive. If
1090          * they do, move the primitive's utilization values to the container.
1091          *
1092          * @TODO This means that bundles without an inner primitive can't have
1093          * utilization. An alternative might be to allow utilization values in
1094          * the top-level bundle XML in the schema, and copy those to each
1095          * container.
1096          */
1097         if (replica->child != NULL) {
1098             GHashTable *empty = replica->container->utilization;
1099 
1100             replica->container->utilization = replica->child->utilization;
1101             replica->child->utilization = empty;
1102         }
1103     }
1104 
1105     if (bundle_data->child) {
1106         rsc->children = g_list_append(rsc->children, bundle_data->child);
1107     }
1108     return TRUE;
1109 }
1110 
1111 static int
1112 replica_resource_active(pe_resource_t *rsc, gboolean all)
     /*  */
1113 {
1114     if (rsc) {
1115         gboolean child_active = rsc->fns->active(rsc, all);
1116 
1117         if (child_active && !all) {
1118             return TRUE;
1119         } else if (!child_active && all) {
1120             return FALSE;
1121         }
1122     }
1123     return -1;
1124 }
1125 
1126 gboolean
1127 pe__bundle_active(pe_resource_t *rsc, gboolean all)
     /*  */
1128 {
1129     pe__bundle_variant_data_t *bundle_data = NULL;
1130     GList *iter = NULL;
1131 
1132     get_bundle_variant_data(bundle_data, rsc);
1133     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1134         pe__bundle_replica_t *replica = iter->data;
1135         int rsc_active;
1136 
1137         rsc_active = replica_resource_active(replica->ip, all);
1138         if (rsc_active >= 0) {
1139             return (gboolean) rsc_active;
1140         }
1141 
1142         rsc_active = replica_resource_active(replica->child, all);
1143         if (rsc_active >= 0) {
1144             return (gboolean) rsc_active;
1145         }
1146 
1147         rsc_active = replica_resource_active(replica->container, all);
1148         if (rsc_active >= 0) {
1149             return (gboolean) rsc_active;
1150         }
1151 
1152         rsc_active = replica_resource_active(replica->remote, all);
1153         if (rsc_active >= 0) {
1154             return (gboolean) rsc_active;
1155         }
1156     }
1157 
1158     /* If "all" is TRUE, we've already checked that no resources were inactive,
1159      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1160      * so return FALSE.
1161      */
1162     return all;
1163 }
1164 
1165 /*!
1166  * \internal
1167  * \brief Find the bundle replica corresponding to a given node
1168  *
1169  * \param[in] bundle  Top-level bundle resource
1170  * \param[in] node    Node to search for
1171  *
1172  * \return Bundle replica if found, NULL otherwise
1173  */
1174 pe_resource_t *
1175 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
     /*  */
1176 {
1177     pe__bundle_variant_data_t *bundle_data = NULL;
1178     CRM_ASSERT(bundle && node);
1179 
1180     get_bundle_variant_data(bundle_data, bundle);
1181     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1182          gIter = gIter->next) {
1183         pe__bundle_replica_t *replica = gIter->data;
1184 
1185         CRM_ASSERT(replica && replica->node);
1186         if (replica->node->details == node->details) {
1187             return replica->child;
1188         }
1189     }
1190     return NULL;
1191 }
1192 
1193 /*!
1194  * \internal
1195  * \deprecated This function will be removed in a future release
1196  */
1197 static void
1198 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1199                   void *print_data)
1200 {
1201     if (rsc != NULL) {
1202         if (options & pe_print_html) {
1203             status_print("<li>");
1204         }
1205         rsc->fns->print(rsc, pre_text, options, print_data);
1206         if (options & pe_print_html) {
1207             status_print("</li>\n");
1208         }
1209     }
1210 }
1211 
1212 /*!
1213  * \internal
1214  * \deprecated This function will be removed in a future release
1215  */
1216 static void
1217 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1218                  void *print_data)
1219 {
1220     pe__bundle_variant_data_t *bundle_data = NULL;
1221     char *child_text = NULL;
1222     CRM_CHECK(rsc != NULL, return);
1223 
1224     if (pre_text == NULL) {
1225         pre_text = "";
1226     }
1227     child_text = crm_strdup_printf("%s        ", pre_text);
1228 
1229     get_bundle_variant_data(bundle_data, rsc);
1230 
1231     status_print("%s<bundle ", pre_text);
1232     status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
1233     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1234     status_print("image=\"%s\" ", bundle_data->image);
1235     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
1236     status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
1237     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
1238     status_print(">\n");
1239 
1240     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1241          gIter = gIter->next) {
1242         pe__bundle_replica_t *replica = gIter->data;
1243 
1244         CRM_ASSERT(replica);
1245         status_print("%s    <replica " XML_ATTR_ID "=\"%d\">\n",
1246                      pre_text, replica->offset);
1247         print_rsc_in_list(replica->ip, child_text, options, print_data);
1248         print_rsc_in_list(replica->child, child_text, options, print_data);
1249         print_rsc_in_list(replica->container, child_text, options, print_data);
1250         print_rsc_in_list(replica->remote, child_text, options, print_data);
1251         status_print("%s    </replica>\n", pre_text);
1252     }
1253     status_print("%s</bundle>\n", pre_text);
1254     free(child_text);
1255 }
1256 
1257 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t *", "GList *", "GList *")
     /*  */
1258 int
1259 pe__bundle_xml(pcmk__output_t *out, va_list args)
1260 {
1261     uint32_t show_opts = va_arg(args, uint32_t);
1262     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1263     GList *only_node = va_arg(args, GList *);
1264     GList *only_rsc = va_arg(args, GList *);
1265 
1266     pe__bundle_variant_data_t *bundle_data = NULL;
1267     int rc = pcmk_rc_no_output;
1268     gboolean printed_header = FALSE;
1269     gboolean print_everything = TRUE;
1270 
1271     const char *desc = NULL;
1272 
1273     CRM_ASSERT(rsc != NULL);
1274     
1275     get_bundle_variant_data(bundle_data, rsc);
1276 
1277     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1278         return rc;
1279     }
1280 
1281     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1282 
1283     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1284          gIter = gIter->next) {
1285         pe__bundle_replica_t *replica = gIter->data;
1286         char *id = NULL;
1287         gboolean print_ip, print_child, print_ctnr, print_remote;
1288 
1289         CRM_ASSERT(replica);
1290 
1291         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1292             continue;
1293         }
1294 
1295         print_ip = replica->ip != NULL &&
1296                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1297         print_child = replica->child != NULL &&
1298                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1299         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1300         print_remote = replica->remote != NULL &&
1301                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1302 
1303         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1304             continue;
1305         }
1306 
1307         if (!printed_header) {
1308             printed_header = TRUE;
1309 
1310             desc = pe__resource_description(rsc, show_opts);
1311 
1312             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
1313                      "id", rsc->id,
1314                      "type", container_agent_str(bundle_data->agent_type),
1315                      "image", bundle_data->image,
1316                      "unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
1317                      "maintenance", pe__rsc_bool_str(rsc, pe_rsc_maintenance),
1318                      "managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
1319                      "failed", pe__rsc_bool_str(rsc, pe_rsc_failed),
1320                      "description", desc);
1321             CRM_ASSERT(rc == pcmk_rc_ok);
1322         }
1323 
1324         id = pcmk__itoa(replica->offset);
1325         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1326         free(id);
1327         CRM_ASSERT(rc == pcmk_rc_ok);
1328 
1329         if (print_ip) {
1330             out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
1331                          replica->ip, only_node, only_rsc);
1332         }
1333 
1334         if (print_child) {
1335             out->message(out, crm_map_element_name(replica->child->xml), show_opts,
1336                          replica->child, only_node, only_rsc);
1337         }
1338 
1339         if (print_ctnr) {
1340             out->message(out, crm_map_element_name(replica->container->xml), show_opts,
1341                          replica->container, only_node, only_rsc);
1342         }
1343 
1344         if (print_remote) {
1345             out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
1346                          replica->remote, only_node, only_rsc);
1347         }
1348 
1349         pcmk__output_xml_pop_parent(out); // replica
1350     }
1351 
1352     if (printed_header) {
1353         pcmk__output_xml_pop_parent(out); // bundle
1354     }
1355 
1356     return rc;
1357 }
1358 
1359 static void
1360 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1361                                pe_node_t *node, uint32_t show_opts)
1362 {
1363     pe_resource_t *rsc = replica->child;
1364 
1365     int offset = 0;
1366     char buffer[LINE_MAX];
1367 
1368     if(rsc == NULL) {
1369         rsc = replica->container;
1370     }
1371 
1372     if (replica->remote) {
1373         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1374                            rsc_printable_id(replica->remote));
1375     } else {
1376         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1377                            rsc_printable_id(replica->container));
1378     }
1379     if (replica->ipaddr) {
1380         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1381                            replica->ipaddr);
1382     }
1383 
1384     pe__common_output_html(out, rsc, buffer, node, show_opts);
1385 }
1386 
1387 /*!
1388  * \internal
1389  * \brief Get a string describing a resource's unmanaged state or lack thereof
1390  *
1391  * \param[in] rsc  Resource to describe
1392  *
1393  * \return A string indicating that a resource is in maintenance mode or
1394  *         otherwise unmanaged, or an empty string otherwise
1395  */
1396 static const char *
1397 get_unmanaged_str(const pe_resource_t *rsc)
     /*  */
1398 {
1399     if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
1400         return " (maintenance)";
1401     }
1402     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
1403         return " (unmanaged)";
1404     }
1405     return "";
1406 }
1407 
1408 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t *", "GList *", "GList *")
     /*  */
1409 int
1410 pe__bundle_html(pcmk__output_t *out, va_list args)
1411 {
1412     uint32_t show_opts = va_arg(args, uint32_t);
1413     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1414     GList *only_node = va_arg(args, GList *);
1415     GList *only_rsc = va_arg(args, GList *);
1416 
1417     const char *desc = NULL;
1418     pe__bundle_variant_data_t *bundle_data = NULL;
1419     int rc = pcmk_rc_no_output;
1420     gboolean print_everything = TRUE;
1421 
1422     CRM_ASSERT(rsc != NULL);
1423 
1424     get_bundle_variant_data(bundle_data, rsc);
1425 
1426     desc = pe__resource_description(rsc, show_opts);
1427 
1428     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1429         return rc;
1430     }
1431 
1432     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1433 
1434     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1435          gIter = gIter->next) {
1436         pe__bundle_replica_t *replica = gIter->data;
1437         gboolean print_ip, print_child, print_ctnr, print_remote;
1438 
1439         CRM_ASSERT(replica);
1440 
1441         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1442             continue;
1443         }
1444 
1445         print_ip = replica->ip != NULL &&
1446                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1447         print_child = replica->child != NULL &&
1448                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1449         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1450         print_remote = replica->remote != NULL &&
1451                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1452 
1453         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1454             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1455             /* The text output messages used below require pe_print_implicit to
1456              * be set to do anything.
1457              */
1458             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1459 
1460             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1461                                      (bundle_data->nreplicas > 1)? " set" : "",
1462                                      rsc->id, bundle_data->image,
1463                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1464                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1465                                      get_unmanaged_str(rsc));
1466 
1467             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1468                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1469             }
1470 
1471             if (print_ip) {
1472                 out->message(out, crm_map_element_name(replica->ip->xml),
1473                              new_show_opts, replica->ip, only_node, only_rsc);
1474             }
1475 
1476             if (print_child) {
1477                 out->message(out, crm_map_element_name(replica->child->xml),
1478                              new_show_opts, replica->child, only_node, only_rsc);
1479             }
1480 
1481             if (print_ctnr) {
1482                 out->message(out, crm_map_element_name(replica->container->xml),
1483                              new_show_opts, replica->container, only_node, only_rsc);
1484             }
1485 
1486             if (print_remote) {
1487                 out->message(out, crm_map_element_name(replica->remote->xml),
1488                              new_show_opts, replica->remote, only_node, only_rsc);
1489             }
1490 
1491             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1492                 out->end_list(out);
1493             }
1494         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1495             continue;
1496         } else {
1497             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1498                                      (bundle_data->nreplicas > 1)? " set" : "",
1499                                      rsc->id, bundle_data->image,
1500                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1501                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1502                                      get_unmanaged_str(rsc));
1503 
1504             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1505                                            show_opts);
1506         }
1507     }
1508 
1509     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1510     return rc;
1511 }
1512 
1513 static void
1514 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1515                                pe_node_t *node, uint32_t show_opts)
1516 {
1517     const pe_resource_t *rsc = replica->child;
1518 
1519     int offset = 0;
1520     char buffer[LINE_MAX];
1521 
1522     if(rsc == NULL) {
1523         rsc = replica->container;
1524     }
1525 
1526     if (replica->remote) {
1527         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1528                            rsc_printable_id(replica->remote));
1529     } else {
1530         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1531                            rsc_printable_id(replica->container));
1532     }
1533     if (replica->ipaddr) {
1534         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1535                            replica->ipaddr);
1536     }
1537 
1538     pe__common_output_text(out, rsc, buffer, node, show_opts);
1539 }
1540 
1541 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t *", "GList *", "GList *")
     /*  */
1542 int
1543 pe__bundle_text(pcmk__output_t *out, va_list args)
1544 {
1545     uint32_t show_opts = va_arg(args, uint32_t);
1546     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1547     GList *only_node = va_arg(args, GList *);
1548     GList *only_rsc = va_arg(args, GList *);
1549 
1550     const char *desc = NULL;
1551     pe__bundle_variant_data_t *bundle_data = NULL;
1552     int rc = pcmk_rc_no_output;
1553     gboolean print_everything = TRUE;
1554 
1555     desc = pe__resource_description(rsc, show_opts);
1556     
1557     get_bundle_variant_data(bundle_data, rsc);
1558 
1559     CRM_ASSERT(rsc != NULL);
1560 
1561     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1562         return rc;
1563     }
1564 
1565     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1566 
1567     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1568          gIter = gIter->next) {
1569         pe__bundle_replica_t *replica = gIter->data;
1570         gboolean print_ip, print_child, print_ctnr, print_remote;
1571 
1572         CRM_ASSERT(replica);
1573 
1574         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1575             continue;
1576         }
1577 
1578         print_ip = replica->ip != NULL &&
1579                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1580         print_child = replica->child != NULL &&
1581                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1582         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1583         print_remote = replica->remote != NULL &&
1584                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1585 
1586         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1587             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1588             /* The text output messages used below require pe_print_implicit to
1589              * be set to do anything.
1590              */
1591             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1592 
1593             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1594                                      (bundle_data->nreplicas > 1)? " set" : "",
1595                                      rsc->id, bundle_data->image,
1596                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1597                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1598                                      get_unmanaged_str(rsc));
1599 
1600             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1601                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1602             }
1603 
1604             out->begin_list(out, NULL, NULL, NULL);
1605 
1606             if (print_ip) {
1607                 out->message(out, crm_map_element_name(replica->ip->xml),
1608                              new_show_opts, replica->ip, only_node, only_rsc);
1609             }
1610 
1611             if (print_child) {
1612                 out->message(out, crm_map_element_name(replica->child->xml),
1613                              new_show_opts, replica->child, only_node, only_rsc);
1614             }
1615 
1616             if (print_ctnr) {
1617                 out->message(out, crm_map_element_name(replica->container->xml),
1618                              new_show_opts, replica->container, only_node, only_rsc);
1619             }
1620 
1621             if (print_remote) {
1622                 out->message(out, crm_map_element_name(replica->remote->xml),
1623                              new_show_opts, replica->remote, only_node, only_rsc);
1624             }
1625 
1626             out->end_list(out);
1627         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1628             continue;
1629         } else {
1630             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1631                                      (bundle_data->nreplicas > 1)? " set" : "",
1632                                      rsc->id, bundle_data->image,
1633                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1634                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1635                                      get_unmanaged_str(rsc));
1636 
1637             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1638                                            show_opts);
1639         }
1640     }
1641 
1642     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1643     return rc;
1644 }
1645 
1646 /*!
1647  * \internal
1648  * \deprecated This function will be removed in a future release
1649  */
1650 static void
1651 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1652                      long options, void *print_data)
1653 {
1654     pe_node_t *node = NULL;
1655     pe_resource_t *rsc = replica->child;
1656 
1657     int offset = 0;
1658     char buffer[LINE_MAX];
1659 
1660     if(rsc == NULL) {
1661         rsc = replica->container;
1662     }
1663 
1664     if (replica->remote) {
1665         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1666                            rsc_printable_id(replica->remote));
1667     } else {
1668         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1669                            rsc_printable_id(replica->container));
1670     }
1671     if (replica->ipaddr) {
1672         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1673                            replica->ipaddr);
1674     }
1675 
1676     node = pe__current_node(replica->container);
1677     common_print(rsc, pre_text, buffer, node, options, print_data);
1678 }
1679 
1680 /*!
1681  * \internal
1682  * \deprecated This function will be removed in a future release
1683  */
1684 void
1685 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1686                  void *print_data)
1687 {
1688     pe__bundle_variant_data_t *bundle_data = NULL;
1689     char *child_text = NULL;
1690     CRM_CHECK(rsc != NULL, return);
1691 
1692     if (options & pe_print_xml) {
1693         bundle_print_xml(rsc, pre_text, options, print_data);
1694         return;
1695     }
1696 
1697     get_bundle_variant_data(bundle_data, rsc);
1698 
1699     if (pre_text == NULL) {
1700         pre_text = " ";
1701     }
1702 
1703     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1704                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1705                  rsc->id, bundle_data->image,
1706                  pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1707                  pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1708     if (options & pe_print_html) {
1709         status_print("<br />\n<ul>\n");
1710     }
1711 
1712 
1713     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1714          gIter = gIter->next) {
1715         pe__bundle_replica_t *replica = gIter->data;
1716 
1717         CRM_ASSERT(replica);
1718         if (options & pe_print_html) {
1719             status_print("<li>");
1720         }
1721 
1722         if (pcmk_is_set(options, pe_print_implicit)) {
1723             child_text = crm_strdup_printf("     %s", pre_text);
1724             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1725                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1726             }
1727             if (options & pe_print_html) {
1728                 status_print("<br />\n<ul>\n");
1729             }
1730             print_rsc_in_list(replica->ip, child_text, options, print_data);
1731             print_rsc_in_list(replica->container, child_text, options, print_data);
1732             print_rsc_in_list(replica->remote, child_text, options, print_data);
1733             print_rsc_in_list(replica->child, child_text, options, print_data);
1734             if (options & pe_print_html) {
1735                 status_print("</ul>\n");
1736             }
1737         } else {
1738             child_text = crm_strdup_printf("%s  ", pre_text);
1739             print_bundle_replica(replica, child_text, options, print_data);
1740         }
1741         free(child_text);
1742 
1743         if (options & pe_print_html) {
1744             status_print("</li>\n");
1745         }
1746     }
1747     if (options & pe_print_html) {
1748         status_print("</ul>\n");
1749     }
1750 }
1751 
1752 static void
1753 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1754 {
1755     if (replica == NULL) {
1756         return;
1757     }
1758 
1759     if (replica->node) {
1760         free(replica->node);
1761         replica->node = NULL;
1762     }
1763 
1764     if (replica->ip) {
1765         free_xml(replica->ip->xml);
1766         replica->ip->xml = NULL;
1767         replica->ip->fns->free(replica->ip);
1768         replica->ip = NULL;
1769     }
1770     if (replica->container) {
1771         free_xml(replica->container->xml);
1772         replica->container->xml = NULL;
1773         replica->container->fns->free(replica->container);
1774         replica->container = NULL;
1775     }
1776     if (replica->remote) {
1777         free_xml(replica->remote->xml);
1778         replica->remote->xml = NULL;
1779         replica->remote->fns->free(replica->remote);
1780         replica->remote = NULL;
1781     }
1782     free(replica->ipaddr);
1783     free(replica);
1784 }
1785 
1786 void
1787 pe__free_bundle(pe_resource_t *rsc)
     /*  */
1788 {
1789     pe__bundle_variant_data_t *bundle_data = NULL;
1790     CRM_CHECK(rsc != NULL, return);
1791 
1792     get_bundle_variant_data(bundle_data, rsc);
1793     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
1794 
1795     free(bundle_data->prefix);
1796     free(bundle_data->image);
1797     free(bundle_data->control_port);
1798     free(bundle_data->host_network);
1799     free(bundle_data->host_netmask);
1800     free(bundle_data->ip_range_start);
1801     free(bundle_data->container_network);
1802     free(bundle_data->launcher_options);
1803     free(bundle_data->container_command);
1804     g_free(bundle_data->container_host_options);
1805 
1806     g_list_free_full(bundle_data->replicas,
1807                      (GDestroyNotify) free_bundle_replica);
1808     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1809     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1810     g_list_free(rsc->children);
1811 
1812     if(bundle_data->child) {
1813         free_xml(bundle_data->child->xml);
1814         bundle_data->child->xml = NULL;
1815         bundle_data->child->fns->free(bundle_data->child);
1816     }
1817     common_free(rsc);
1818 }
1819 
1820 enum rsc_role_e
1821 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
     /*  */
1822 {
1823     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
1824     return container_role;
1825 }
1826 
1827 /*!
1828  * \brief Get the number of configured replicas in a bundle
1829  *
1830  * \param[in] rsc  Bundle resource
1831  *
1832  * \return Number of configured replicas, or 0 on error
1833  */
1834 int
1835 pe_bundle_replicas(const pe_resource_t *rsc)
     /*  */
1836 {
1837     if ((rsc == NULL) || (rsc->variant != pe_container)) {
1838         return 0;
1839     } else {
1840         pe__bundle_variant_data_t *bundle_data = NULL;
1841 
1842         get_bundle_variant_data(bundle_data, rsc);
1843         return bundle_data->nreplicas;
1844     }
1845 }
1846 
1847 void
1848 pe__count_bundle(pe_resource_t *rsc)
     /*  */
1849 {
1850     pe__bundle_variant_data_t *bundle_data = NULL;
1851 
1852     get_bundle_variant_data(bundle_data, rsc);
1853     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
1854         pe__bundle_replica_t *replica = item->data;
1855 
1856         if (replica->ip) {
1857             replica->ip->fns->count(replica->ip);
1858         }
1859         if (replica->child) {
1860             replica->child->fns->count(replica->child);
1861         }
1862         if (replica->container) {
1863             replica->container->fns->count(replica->container);
1864         }
1865         if (replica->remote) {
1866             replica->remote->fns->count(replica->remote);
1867         }
1868     }
1869 }
1870 
1871 gboolean
1872 pe__bundle_is_filtered(const pe_resource_t *rsc, GList *only_rsc,
     /*  */
1873                        gboolean check_parent)
1874 {
1875     gboolean passes = FALSE;
1876     pe__bundle_variant_data_t *bundle_data = NULL;
1877 
1878     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
1879         passes = TRUE;
1880     } else {
1881         get_bundle_variant_data(bundle_data, rsc);
1882 
1883         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
1884             pe__bundle_replica_t *replica = gIter->data;
1885 
1886             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
1887                 passes = TRUE;
1888                 break;
1889             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
1890                 passes = TRUE;
1891                 break;
1892             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
1893                 passes = TRUE;
1894                 break;
1895             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
1896                 passes = TRUE;
1897                 break;
1898             }
1899         }
1900     }
1901 
1902     return !passes;
1903 }
1904 
1905 /*!
1906  * \internal
1907  * \brief Get a list of a bundle's containers
1908  *
1909  * \param[in] bundle  Bundle resource
1910  *
1911  * \return Newly created list of \p bundle's containers
1912  * \note It is the caller's responsibility to free the result with
1913  *       g_list_free().
1914  */
1915 GList *
1916 pe__bundle_containers(const pe_resource_t *bundle)
     /*  */
1917 {
1918     GList *containers = NULL;
1919     const pe__bundle_variant_data_t *data = NULL;
1920 
1921     get_bundle_variant_data(data, bundle);
1922     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
1923         pe__bundle_replica_t *replica = iter->data;
1924 
1925         containers = g_list_append(containers, replica->container);
1926     }
1927     return containers;
1928 }
1929 
1930 // Bundle implementation of resource_object_functions_t:active_node()
1931 pe_node_t *
1932 pe__bundle_active_node(const pe_resource_t *rsc, unsigned int *count_all,
     /*  */
1933                        unsigned int *count_clean)
1934 {
1935     pe_node_t *active = NULL;
1936     pe_node_t *node = NULL;
1937     pe_resource_t *container = NULL;
1938     GList *containers = NULL;
1939     GList *iter = NULL;
1940     GHashTable *nodes = NULL;
1941     const pe__bundle_variant_data_t *data = NULL;
1942 
1943     if (count_all != NULL) {
1944         *count_all = 0;
1945     }
1946     if (count_clean != NULL) {
1947         *count_clean = 0;
1948     }
1949     if (rsc == NULL) {
1950         return NULL;
1951     }
1952 
1953     /* For the purposes of this method, we only care about where the bundle's
1954      * containers are active, so build a list of active containers.
1955      */
1956     get_bundle_variant_data(data, rsc);
1957     for (iter = data->replicas; iter != NULL; iter = iter->next) {
1958         pe__bundle_replica_t *replica = iter->data;
1959 
1960         if (replica->container->running_on != NULL) {
1961             containers = g_list_append(containers, replica->container);
1962         }
1963     }
1964     if (containers == NULL) {
1965         return NULL;
1966     }
1967 
1968     /* If the bundle has only a single active container, just use that
1969      * container's method. If live migration is ever supported for bundle
1970      * containers, this will allow us to prefer the migration source when there
1971      * is only one container and it is migrating. For now, this just lets us
1972      * avoid creating the nodes table.
1973      */
1974     if (pcmk__list_of_1(containers)) {
1975         container = containers->data;
1976         node = container->fns->active_node(container, count_all, count_clean);
1977         g_list_free(containers);
1978         return node;
1979     }
1980 
1981     // Add all containers' active nodes to a hash table (for uniqueness)
1982     nodes = g_hash_table_new(NULL, NULL);
1983     for (iter = containers; iter != NULL; iter = iter->next) {
1984         container = iter->data;
1985 
1986         for (GList *node_iter = container->running_on; node_iter != NULL;
1987              node_iter = node_iter->next) {
1988             node = node_iter->data;
1989 
1990             // If insert returns true, we haven't counted this node yet
1991             if (g_hash_table_insert(nodes, (gpointer) node->details,
1992                                     (gpointer) node)
1993                 && !pe__count_active_node(rsc, node, &active, count_all,
1994                                           count_clean)) {
1995                 goto done;
1996             }
1997         }
1998     }
1999 
2000 done:
2001     g_list_free(containers);
2002     g_hash_table_destroy(nodes);
2003     return active;
2004 }

/* */