...
There are three main operations which can be carried out on UDSPs either from the command line or YAML configuraitonconfiguration: add, delete, show.
Add
The UI allows adding a new rule. With the use of the idx optional parameter, the admin can specifiy where in the rule chain the new rule should be added. By default the rule is appended to the list. Any other value will result in inserting the rule in that position.
...
Show
The UI allows showing exisitng existng UDSPs. The format of the YAML output is as follows:
...
All policies are stored in kernel space. All logic to add, delete and match policies will be implemented in kernel space. This complicates the kernel space processing. Arguably, policy maintenance logic is not core to LNet functionality. What is core is the ability to select source and destination networks and NIDs in accordance with user definitions. However, the kernel is able to manage policies much easier and with less potential race conditions than user space.
Design Principles
Rule Storage
Rules shall be defined through user space and passed to the by administrators either via LNet command line utility, lnetctl, or via YAML configuration file. lnetctl parses the UDSP and stores it in an intermediary format, which will be flattened and passed down to the kernel LNet module. LNet shall store these rules on a policy list. Once policies are added to LNet they will be applied on existing networks, NIDs and routers. The advantage of this approach is that rules are not strictly tied to the internal constructs, IE networks, NIDs or routers, but can be applied whenever the internal constructs are created and if the internal constructs are deleted then they remain and can be automatically applied at a future time.
...
Peers are normally not created explicitly by the adminadministrators. The ULP requests to send a message to a peer or the node receives an unsolicited message from a peer which results in creating a peer construct in LNet. It is feasible, especially for router policies, where we want to associate a specific to have a UDSP which associates a set of routers to clients within with in a user defined rangespecific range with a set of optimal routers. Having the policies and matching occur in kernel aids in fulfilling this requirement.
Rule Application
Performance needs to be taken into account with this feature. It is not feasible to traverse the policy lists on every send operation. This will add unnecessary overhead. When rules are applied they have to be "flattened" to the constructs they impact. For example, a Network Rule is added as follows: o2ib priority 0. This rule gives priority for using o2ib network for sending. A priority field in the network will be added. This will be set to 0 for the o2ib network. As we traverse the networks in the selection algorithm, which is part of the current code, the priority field will be compared. This is a more optimal approach than examining the policies on every send to see if it we get any matches.
In Kernel Structures
| Code Block |
|---|
/* lnet structure will keep a list of UDSPs */
struct lnet {
...
list_head ln_udsp_list;
...
}
/* each NID range is defined as net_id and an ip range */
struct lnet_ud_nid_descr {
__u32 ud_net_id;
list_head ud_ip_range;
}
/* UDSP action types */
enum lnet_udsp_action_type udsp_action_type {
EN_LNET_UDSP_ACTION_PRIORITY = 0,
EN_LNET_UDSP_ACTION_NONE = 1,
}
/*
* a UDSP rule can have up to three user defined NID descriptors
* - src: defines the local NID range for the rule
* - dst: defines the peer NID range for the rule
* - rte: defines the router NID range for the rule
*
* An action union defines the action to take when the rule
* is matched
*/
struct lnet_udsp {
list_head udsp_on_list;
lnet_ud_nid_descr *udsp_src;
lnet_ud_nid_describe *udsp_dst;
lnet_ud_nid_descr *udsp_rte;
enum lnet_udsp_action_type udsp_action_type;
union udsp_action {
__u32 udsp_priority;
};
}
/* The rules are flattened in the LNet structures as shown below */
struct lnet_net {
...
/* defines the relative priority of this net compared to others in the system */
__u32 net_priority;
...
}
struct lnet_ni {
...
/* defines the relative priority of this NI compared to other NIs in the net */
__u32 ni_priority;
...
}
struct lnet_peer_ni {
...
/* defines the relative peer_ni priority compared to other peer_nis in the peer */
__u32 lpni_priority;
/* defines the list of local NID(s) (>=1) which should be used as the source */
union lpni_pref {
lnet_nid_t nid;
lnet_nid_t *nids;
}
/* defines the list of router NID(s) to be used when sending to this peer NI */
lnet_nid_t *lpni_rte_nids;
...
}
/* UDSPs will be passed to the kernel via IOCTL */
#define IOC_LIBCFS_ADD_UDSP _IOWR(IOC_LIBCFS_TYPE, 106, IOCTL_CONFIG_SIZE)
/* UDSP will be grabbed from the kernel via IOCTL
#define IOC_LIBCFS_GET_UDSP _IOWR(IOC_LIBCFS_TYPE, 106, IOCTL_CONFIG_SIZE) |
...
Kernel Selection Algorithm Modifications
| Code Block |
|---|
/* * Whenselect aan UDSPNI rulefrom associatesthe localNets NIs with remotehighest NIs, the list of local NIs NIDs * is flattened to a list in the associated peer_NI. When selecting a peer NI, the * peer NI with the corresponding preferred local NI is selected. */ bool lnet_peer_is_pref_nid_lockedpriority */ struct lnet_ni * lnet_find_best_ni_on_local_net(struct lnet_peer_ni *lpnipeer, lnet_nid_t nidint md_cpt) { ... } static struct lnet_peer_ni * lnet_select_peer_ni(struct lnet_send_data *sd, struct lnet_peer *peer, list_for_each_entry(peer_net, &peer->lp_peer_nets, lpn_peer_nets) { ... struct lnet_peer_net *peer_net) { ... ni_is_pref; net = lnet_peer_is_pref_nidget_net_locked(lpni, best_ni->ni_nidpeer_net->lpn_net_id); lpni_prio = lpni->lpni_priority; if (lpni_healthv < best_lpni_healthv!net) continue; /* * look selectonly at the NINIs with the highest priority and disregard * nets which have lower priority. Nets with equal priority are * examined and the best_ni is selected from amongst them. */ else if lpni_prio > best_lpni_prio) continue; else if (lpni_prio < best_lpni_prio) best_lpni_prio = lpni_prio; /* * select the NI which has the best_ni's NID in its preferred list */ else if (!preferred && ni_is_pref) preferred = true; ... } static struct lnet_ni * lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, struct lnet_peer *peer, net_prio = net->net_priority; if (net_prio > best_net_prio) continue; else if (net_prio < best_net_prio) { best_net_prio = net_prio; best_ni = NULL; } best_ni = lnet_find_best_ni_on_spec_net(best_ni, peer, best_peer_net, md_cpt, false); ... } ... } /* * select the NI with the highest priority */ static struct lnet_ni * lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *best_ni, struct lnet_peer *peer, struct lnet_peer_net *peer_net, int md_cpt) { ... ni_prio = ni->ni_priority; if (ni_fatal) { continue; } else if (ni_healthv < best_healthv) { continue; } else if (ni_healthv > best_healthv) { best_healthv = ni_healthv; if (distance < shortest_distance) shortest_distance = distance; /* * if this NI is lower in priority than the one already set then discard it * otherwise use it and set the best prioirty so far to this NI's. */ } else if ni_prio > best_ni_prio) { continue; } else if (ni_prio < best_ni_prio) best_ni_prio = ni_prio; } ... } /* * When a UDSP rule associates local NIs with remote NIs, the list of local NIs NIDs * is flattened to a list in the associated peer_NI. When selecting a peer NI, the * peer NI with the corresponding preferred local NI is selected. */ bool lnet_peer_is_pref_nid_locked(struct lnet_peer_ni *lpni, lnet_nid_t nid) { ... } /* * select the peer NI with the highest priority first and then the * preferred one */ static struct lnet_peer_ni * lnet_select_peer_ni(struct lnet_send_data *sd, struct lnet_peer *peer, struct lnet_peer_net *peer_net, int md_cpt) { ... ni_prio = ni->ni_priority; if (ni_fatal) { continue; } else if (ni_healthv < best_healthv) { continue; } else if (ni_healthv > best_healthv) { best_healthv = ni_healthv; if (distance < shortest_distance) shortest_distance = distanceni_is_pref = lnet_peer_is_pref_nid_locked(lpni, best_ni->ni_nid); lpni_prio = lpni->lpni_priority; if (lpni_healthv < best_lpni_healthv) continue; /* * if this NI is lower in priority than the one already set then discard it * otherwise use it and set * select the bestNI prioirtywith sothe far to this NI'shighest priority. */ } else if nilpni_prio > best_nilpni_prio) { continue; } else if (nilpni_prio < best_nilpni_prio) best_nilpni_prio = nilpni_prio; } /* * select the NI which has the best_ni's NID in its preferred list */ else if (!preferred && ni_is_pref) preferred = true; ... } |
Rule Marshaling
Rule Structure
Selection policy rules are comprised of two parts:
...