{
  "AWSTemplateFormatVersion" : "2010-09-09",
  "Description" : "Create cluster of Lustre servers with persistent EBS volumes.",
  "Parameters" : {
    "KeyName" : {
      "Description" : "Name of and existing EC2 KeyPair to enable SSH access to the instance",
      "Type" : "String"
    },
    "FsName" : {
      "Description" : "Name of the lustre filesystem.",
      "Type" : "String",
      "Default" : "scratch"
    },
    "MgsInstanceType" : {
      "Description" : "Instance type used for Lustre MGS servers",
      "Type" : "String",
      "Default" : "m1.xlarge"
    },
    "MgtVolumeSize" : {
      "Description" : "Size of EBS volumes to use for MGS.",
      "Type" : "Number",
      "Default" : "10"
    },
    "OssInstanceType" : {
      "Description" : "Instance type used for Lustre OSS servers",
      "Type" : "String",
      "Default" : "m1.xlarge"
    },
    "OssCount" : {
      "Description" : "Number of OSS instances.",
      "Type" : "Number",
      "Default" : "1"
    },
    "OstVolumeSize" : {
      "Description" : "Size of EBS volumes to use for OSTs.",
      "Type" : "Number",
      "Default" : "40"
    },
    "OssUsingEbs" : {
      "Description" : "EBS volumes are created in the CFN template.",
      "Type" : "String",
      "Default" : "true"
    },
    "MdsInstanceType" : {
      "Description" : "Instance type used for Lustre servers",
      "Type" : "String",
      "Default" : "m3.2xlarge"
    },
    "MdsCount" : {
      "Description" : "Number of MDS instances.",
      "Type" : "Number",
      "Default" : "1"
    },
    "MdtVolumeSize" : {
      "Description" : "Size of EBS volumes to use for MDTs. All instances have 5 EBS volumes attached.",
      "Type" : "Number",
      "Default" : "40"
    },
    "MdsUsingEbs" : {
      "Description" : "EBS volumes are created in the CFN template.",
      "Type" : "String",
      "Default" : "true"
    },
    "ClientInstanceType" : {
      "Description" : "Instance type used for Lustre servers",
      "Type" : "String",
      "Default" : "m3.2xlarge"
    },
    "ClientCount" : {
      "Description" : "Number of Client instances.",
      "Type" : "Number",
      "Default" : "2"
    },
    "NATInstanceType" : {
      "Description" : "NAT Device EC2 instance type",
      "Type" : "String",
      "Default" : "m1.small",
      "AllowedValues" : [
        "t1.micro",
        "m1.small",
        "m1.medium",
        "m1.large",
        "m1.xlarge",
        "m2.xlarge",
        "m2.2xlarge",
        "m2.4xlarge",
        "c1.medium",
        "c1.xlarge",
        "cc2.8xlarge"
      ],
      "ConstraintDescription" : "must be a valid EC2 instance type."
    },
    "VpcPrivateCIDR" : {
      "Description" : "CIDR for new private subnet i.e. 10.0.2.0/24",
      "Type" : "String",
      "MinLength" : "9",
      "MaxLength" : "18",
      "Default" : "172.16.12.0/24",
      "AllowedPattern" : "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription" : "must be a valid CIDR range of the form x.x.x.x/x."
    },
    "VpcId" : {
      "Description" : "Id of an exisiting VPC that contains a public subnet i.e. vpc-d54ebebf",
      "Type" : "String",
      "MinLength" : "1",
      "MaxLength" : "64",
      "AllowedPattern" : "vpc-[a-z0-9]*",
      "ConstraintDescription" : "can contain only alphanumeric characters, spaces, dashes and underscores.",
      "Default" : "vpc-"
    },
    "VpcPublicSubnetId" : {
      "Description" : "Id of an exisiting public VPC subnet i.e. subnet-ae4ebec4",
      "Type" : "String",
      "Default" : "subnet-",
      "MinLength" : "1",
      "MaxLength" : "64",
      "AllowedPattern" : "subnet-[a-z0-9]*",
      "ConstraintDescription" : "can contain only alphanumeric characters, spaces, dashes and underscores."
    },
    "SSHFrom" : {
      "Description" : "Lockdown SSH access to the NAT host (default can be accessed from anywhere)",
      "Type" : "String",
      "MinLength" : "9",
      "MaxLength" : "18",

      "Default" : "0.0.0.0/0",
      "AllowedPattern" : "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription" : "must be a valid CIDR range of the form x.x.x.x/x."
    },
    "AccessFrom" : {
      "Description" : "Lockdown access to Lustre services (default is accessiable for 0.0.0.0/0)",
      "Type" : "String",
      "MinLength" : "9",
      "MaxLength" : "18",
      "Default" : "0.0.0.0/0",
      "AllowedPattern" : "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription" : "must be a valid CIDR range of the form x.x.x.x/x."
    },
    "HTTPFrom" : {
      "Description" : "Lockdown access to Lustre Ganglia on MGS (default is accessiable for 0.0.0.0/0)",
      "Type" : "String",
      "MinLength" : "9",
      "MaxLength" : "18",
      "Default" : "0.0.0.0/0",
      "AllowedPattern" : "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription" : "must be a valid CIDR range of the form x.x.x.x/x."
    }
  },
  "Mappings" : {
    "AWSInstanceType2Arch" : {
      "t1.micro" : {
        "Arch" : "64"
      },
      "m1.small" : {
        "Arch" : "64"
      },
      "m1.medium" : {
        "Arch" : "64"
      },
      "m1.large" : {
        "Arch" : "64"
      },
      "m1.xlarge" : {
        "Arch" : "64"
      },
      "m2.xlarge" : {
        "Arch" : "64"
      },
      "m2.2xlarge" : {
        "Arch" : "64"
      },
      "m2.4xlarge" : {
        "Arch" : "64"
      },
      "c1.medium" : {
        "Arch" : "64"
      },
      "c1.xlarge" : {
        "Arch" : "64"
      },
      "cc1.4xlarge" : {
        "Arch" : "64HVM"
      },
      "cc2.8xlarge" : {
        "Arch" : "64HVM"
      },
      "hs1.8xlarge" : {
        "Arch" : "64HVM"
      },
      "hi1.4xlarge" : {
        "Arch" : "64"
      },
      "m3.xlarge" : {
        "Arch" : "64"
      },
      "m3.2xlarge" : {
        "Arch" : "64"
      }
    },
    "AWSRegionArch2ServerAMI" : {
      "us-east-1" : {
        "64" : "ami-ab104cc2",
        "64HVM" : "ami-dd124eb4"
      },
      "eu-west-1" : {
        "64" : "",
        "64HVM" : ""
      },
      "us-west-1" : {
        "64" : "",
        "64HVM" : "NOT_YET_SUPPORTED"
      },
      "ap-southeast-1" : {
        "64" : "",
        "64HVM" : "NOT_YET_SUPPORTED"
      },
      "ap-northeast-1" : {
        "64" : "",
        "64HVM" : "NOT_YET_SUPPORTED"
      },
      "ap-southeast-2" : {
        "64" : "",
        "64HVM" : "NOT_YET_SUPPORTED"
      },
      "sa-east-1" : {
        "64" : "",
        "64HVM" : "NOT_YET_SUPPORTED"
      },
      "us-west-2" : {
        "64" : "ami-c23ca5f2",
        "64HVM" : "ami-c43ca5f4"
      }
    },
    "InstanceSpotPrice" : {
      "m1.large" : {
        "price" : "0.24"
      },
      "m1.xlarge" : {
        "price" : "0.48"
      },
      "m3.xlarge" : {
        "price" : "0.50"
      },
      "m3.2xlarge" : {
        "price" : "1.00"
      },
      "cc2.8xlarge" : {
        "price" : "1.00"
      }
    },
    "AWSNATAMI" : {
      "us-east-1" : {
        "AMI" : "ami-4f9fee26"
      },
      "us-west-2" : {
        "AMI" : "ami-6d29b85d"
      },
      "us-west-1" : {
        "AMI" : "ami-7850793d"
      },
      "eu-west-1" : {
        "AMI" : "ami-ed352799"
      },
      "ap-southeast-1" : {
        "AMI" : "ami-780a432a"
      },
      "ap-northeast-1" : {
        "AMI" : "ami-5f840e5e"
      },
      "sa-east-1" : {
        "AMI" : "ami-7660c56b"
      }
    }
  },
  "Resources" : {
    "OssWaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle",
      "Properties" : {}
    },
    "OssWaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "DependsOn" : "OSSNodes",
      "Properties" : {
        "Handle" : {
          "Ref" : "OssWaitHandle"
        },
        "Count" : {
          "Ref" : "OssCount"
        },
        "Timeout" : "3600"
      }
    },
    "lustremd" : {
      "Type" : "AWS::DynamoDB::Table",
      "Properties" : {
        "KeySchema" : {
          "HashKeyElement" : {
            "AttributeName" : "id",
            "AttributeType" : "S"
          }
        },
        "ProvisionedThroughput" : {
          "ReadCapacityUnits" : "10",
          "WriteCapacityUnits" : "5"
        }
      }
    },
    "lustremdRoles" : {
      "Type" : "AWS::DynamoDB::Table",
      "Properties" : {
        "KeySchema" : {
          "HashKeyElement" : {
            "AttributeName" : "network_interface_id",
            "AttributeType" : "S"
          }
        },
        "ProvisionedThroughput" : {
          "ReadCapacityUnits" : "10",
          "WriteCapacityUnits" : "5"
        }
      }
    },
    "ClientRole" : {
      "Type" : "AWS::IAM::Role",
      "Properties" : {
        "AssumeRolePolicyDocument" : {
          "Statement" : [
            {
              "Effect" : "Allow",
              "Principal" : {
                "Service" : [
                  "ec2.amazonaws.com"
                ]
              },
              "Action" : [
                "sts:AssumeRole"
              ]
            }
          ]
        },
        "Path" : "/"
      }
    },
    "ServerRole" : {
      "Type" : "AWS::IAM::Role",
      "Properties" : {
        "AssumeRolePolicyDocument" : {
          "Statement" : [
            {
              "Effect" : "Allow",
              "Principal" : {
                "Service" : [
                  "ec2.amazonaws.com"
                ]
              },
              "Action" : [
                "sts:AssumeRole"
              ]
            }
          ]
        },
        "Path" : "/"
      }
    },
    "BasePolicy" : {
      "Type" : "AWS::IAM::Policy",
      "Properties" : {
        "PolicyName" : "BasePolicy",
        "PolicyDocument" : {
          "Statement" : [
            {
              "Effect" : "Allow",
              "Resource" : "*",
              "Action" : [
                "EC2:DescribeInstances",
                "EC2:DescribeInstanceStatus",
                "EC2:CreateTags",
                "CloudFormation:*",
                "dynamodb:*"
              ]
            }
          ]
        },
        "Roles" : [
          {
            "Ref" : "ServerRole"
          },
          {
            "Ref" : "ClientRole"
          }
        ]
      }
    },
    "ServerPolicy" : {
      "Type" : "AWS::IAM::Policy",
      "Properties" : {
        "PolicyName" : "ServerPolicy",
        "PolicyDocument" : {
          "Statement" : [
            {
              "Action" : [
                "ec2:AttachNetworkInterface",
                "ec2:AttachVolume",
                "ec2:CreateNetworkInterface",
                "ec2:CreateVolume",
                "ec2:DeleteNetworkInterface",
                "ec2:DeleteVolume",
                "ec2:DetachNetworkInterface",
                "ec2:DetachVolume",
                "ec2:DescribeNetworkInterfaces",
                "ec2:DescribeNetworkInterfaceAttribute",
                "ec2:DescribeVolumes",
                "ec2:DescribeVolumeAttribute",
                "ec2:DescribeVolumeStatus"
              ],
              "Effect" : "Allow",
              "Resource" : "*"
            }
          ]
        },
        "Roles" : [
          {
            "Ref" : "ServerRole"
          }
        ]
      }
    },
    "ClientInstanceProfile" : {
      "Type" : "AWS::IAM::InstanceProfile",
      "Properties" : {
        "Path" : "/",
        "Roles" : [
          {
            "Ref" : "ClientRole"
          }
        ]
      }
    },
    "ServerInstanceProfile" : {
      "Type" : "AWS::IAM::InstanceProfile",
      "Properties" : {
        "Path" : "/",
        "Roles" : [
          {
            "Ref" : "ServerRole"
          }
        ]
      }
    },
    "MDSNodes" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "LaunchConfigurationName" : {
          "Ref" : "MDSLaunchConfig"
        },
        "DesiredCapacity" : {
          "Ref" : "MdsCount"
        },
        "Tags" : [
          {
            "Key" : "lustre:type",
            "Value" : "mdt",
            "PropagateAtLaunch" : "true"
          },
          {
            "Key" : "loci:application",
            "Value" : {
              "Ref" : "AWS::StackId"
            },
            "PropagateAtLaunch" : "true"
          }
        ],
        "MaxSize" : {
          "Ref" : "MdsCount"
        },
        "MinSize" : {
          "Ref" : "MdsCount"
        },
        "VPCZoneIdentifier" : [
          {
            "Ref" : "PrivateSubnet"
          }
        ],
        "AvailabilityZones" : [
          {
            "Fn::GetAtt" : [
              "PrivateSubnet",
              "AvailabilityZone"
            ]
          }
        ]
      }
    },
    "MDSLaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config" : {
            "files" : {
              "/etc/loci.conf" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "[default]\n",
                      "FsName: ",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "HA: true\n",
                      "FilesystemTable: ",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "RolesTable: ",
                      {
                        "Ref" : "lustremdRoles"
                      },
                      "\n",
                      "MdsCount: ",
                      {
                        "Ref" : "MdsCount"
                      },
                      "\n",
                      "OssCount: ",
                      {
                        "Ref" : "OssCount"
                      },
                      "\n",
                      "LustreAZ: ",
                      {
                        "Fn::GetAtt" : [
                          "PrivateSubnet",
                          "AvailabilityZone"
                        ]
                      },
                      "\n",
                      "MgtVolumeSize: ",
                      {
                        "Ref" : "MgtVolumeSize"
                      },
                      "\n",
                      "MdtVolumeSize: ",
                      {
                        "Ref" : "MdtVolumeSize"
                      },
                      "\n",
                      "OstVolumeSize: ",
                      {
                        "Ref" : "OstVolumeSize"
                      },
                      "\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              },
              "/etc/cloud.env" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "export FSNAME=",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "export PERSONALITY=MDS\n",
                      "export LUSTREMD=",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "export HA=false\n",
                      "export USING_EBS=",
                      {
                        "Ref" : "MdsUsingEbs"
                      },
                      "\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              }
            }
          }
        }
      },
      "Properties" : {
        "SecurityGroups" : [
          {
            "Ref" : "LustreSecurityGroup"
          }
        ],
        "KeyName" : {
          "Ref" : "KeyName"
        },
        "IamInstanceProfile" : {
          "Ref" : "ServerInstanceProfile"
        },
        "ImageId" : {
          "Fn::FindInMap" : [
            "AWSRegionArch2ServerAMI",
            {
              "Ref" : "AWS::Region"
            },
            {
              "Fn::FindInMap" : [
                "AWSInstanceType2Arch",
                {
                  "Ref" : "MdsInstanceType"
                },
                "Arch"
              ]
            }
          ]
        },
        "InstanceType" : {
          "Ref" : "MdsInstanceType"
        },
        "SpotPrice" : {
          "Fn::FindInMap" : [
            "InstanceSpotPrice",
            {
              "Ref" : "MdsInstanceType"
            },
            "price"
          ]
        },
        "EbsOptimized" : "true",
        "InstanceMonitoring" : "false",
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : [
              "",
              [
                "#!/bin/bash -ex\n",
                "cfn-init -s ",
                {
                  "Ref" : "AWS::StackName"
                },
                " -r MDSLaunchConfig ",
                "   --region ",
                {
                  "Ref" : "AWS::Region"
                },
                "\n",
                "/root/setup-lustre\n"
              ]
            ]
          }
        },
        "BlockDeviceMappings" : [
          {
            "DeviceName" : "/dev/sdf",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdg",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdh",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdi",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdj",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdk",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdl",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdm",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MdtVolumeSize"
              }
            }
          }
        ]
      }
    },
    "OSSNodes" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "LaunchConfigurationName" : {
          "Ref" : "OSSLaunchConfig"
        },
        "DesiredCapacity" : {
          "Ref" : "OssCount"
        },
        "Tags" : [
          {
            "Key" : "lustre:type",
            "Value" : "ost",
            "PropagateAtLaunch" : "true"
          },
          {
            "Key" : "loci:application",
            "Value" : {
              "Ref" : "AWS::StackId"
            },
            "PropagateAtLaunch" : "true"
          }
        ],
        "MaxSize" : {
          "Ref" : "OssCount"
        },
        "MinSize" : {
          "Ref" : "OssCount"
        },
        "VPCZoneIdentifier" : [
          {
            "Ref" : "PrivateSubnet"
          }
        ],
        "AvailabilityZones" : [
          {
            "Fn::GetAtt" : [
              "PrivateSubnet",
              "AvailabilityZone"
            ]
          }
        ]
      }
    },
    "OSSLaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config" : {
            "files" : {
              "/etc/loci.conf" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "[default]\n",
                      "FsName: ",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "HA: true\n",
                      "FilesystemTable: ",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "RolesTable: ",
                      {
                        "Ref" : "lustremdRoles"
                      },
                      "\n",
                      "MdsCount: ",
                      {
                        "Ref" : "MdsCount"
                      },
                      "\n",
                      "OssCount: ",
                      {
                        "Ref" : "OssCount"
                      },
                      "\n",
                      "LustreAZ: ",
                      {
                        "Fn::GetAtt" : [
                          "PrivateSubnet",
                          "AvailabilityZone"
                        ]
                      },
                      "\n",
                      "MgtVolumeSize: ",
                      {
                        "Ref" : "MgtVolumeSize"
                      },
                      "\n",
                      "MdtVolumeSize: ",
                      {
                        "Ref" : "MdtVolumeSize"
                      },
                      "\n",
                      "OstVolumeSize: ",
                      {
                        "Ref" : "OstVolumeSize"
                      },
                      "\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              },
              "/etc/cloud.env" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "export FSNAME=",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "export PERSONALITY=OSS\n",
                      "export USING_EBS=",
                      {
                        "Ref" : "OssUsingEbs"
                      },
                      "\n",
                      "export LUSTREMD=",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "export HA=false\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              }
            }
          }
        }
      },
      "Properties" : {
        "SecurityGroups" : [
          {
            "Ref" : "LustreSecurityGroup"
          }
        ],
        "KeyName" : {
          "Ref" : "KeyName"
        },
        "IamInstanceProfile" : {
          "Ref" : "ServerInstanceProfile"
        },
        "ImageId" : {
          "Fn::FindInMap" : [
            "AWSRegionArch2ServerAMI",
            {
              "Ref" : "AWS::Region"
            },
            {
              "Fn::FindInMap" : [
                "AWSInstanceType2Arch",
                {
                  "Ref" : "OssInstanceType"
                },
                "Arch"
              ]
            }
          ]
        },
        "InstanceType" : {
          "Ref" : "OssInstanceType"
        },
        "SpotPrice" : {
          "Fn::FindInMap" : [
            "InstanceSpotPrice",
            {
              "Ref" : "OssInstanceType"
            },
            "price"
          ]
        },
        "EbsOptimized" : "true",
        "InstanceMonitoring" : "false",
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : [
              "",
              [
                "#!/bin/bash -ex\n",
                "cfn-init -s ",
                {
                  "Ref" : "AWS::StackName"
                },
                " -r OSSLaunchConfig ",
                "   --region ",
                {
                  "Ref" : "AWS::Region"
                },
                "\n",
                "/root/setup-lustre\n",
                "cfn-signal -e $? -r 'OSS setup done' '",
                {
                  "Ref" : "OssWaitHandle"
                },
                "'\n"
              ]
            ]
          }
        },
        "BlockDeviceMappings" : [
          {
            "DeviceName" : "/dev/sdf",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdg",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdh",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdi",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdj",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdk",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdl",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          },
          {
            "DeviceName" : "/dev/sdm",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "OstVolumeSize"
              }
            }
          }
        ]
      }
    },
    "ClientNodes" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "LaunchConfigurationName" : {
          "Ref" : "ClientLaunchConfig"
        },
        "DesiredCapacity" : {
          "Ref" : "ClientCount"
        },
        "Tags" : [
          {
            "Key" : "lustre:type",
            "Value" : "client",
            "PropagateAtLaunch" : "true"
          },
          {
            "Key" : "loci:application",
            "Value" : {
              "Ref" : "AWS::StackId"
            },
            "PropagateAtLaunch" : "true"
          }
        ],
        "MaxSize" : {
          "Ref" : "ClientCount"
        },
        "MinSize" : {
          "Ref" : "ClientCount"
        },
        "VPCZoneIdentifier" : [
          {
            "Ref" : "PrivateSubnet"
          }
        ],
        "AvailabilityZones" : [
          {
            "Fn::GetAtt" : [
              "PrivateSubnet",
              "AvailabilityZone"
            ]
          }
        ]
      }
    },
    "ClientLaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config" : {
            "packages" : {
              "yum" : {
                "dbench" : [],
                "bonnie++" : [],
                "screen" : [],
                "environment-modules" : [],
                "openmpi" : []
              }
            },
            "files" : {
              "/etc/loci.conf" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "[default]\n",
                      "FsName: ",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "HA: true\n",
                      "FilesystemTable: ",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "RolesTable: ",
                      {
                        "Ref" : "lustremdRoles"
                      },
                      "\n",
                      "MdsCount: ",
                      {
                        "Ref" : "MdsCount"
                      },
                      "\n",
                      "OssCount: ",
                      {
                        "Ref" : "OssCount"
                      },
                      "\n",
                      "LustreAZ: ",
                      {
                        "Fn::GetAtt" : [
                          "PrivateSubnet",
                          "AvailabilityZone"
                        ]
                      },
                      "\n",
                      "MgtVolumeSize: ",
                      {
                        "Ref" : "MgtVolumeSize"
                      },
                      "\n",
                      "MdtVolumeSize: ",
                      {
                        "Ref" : "MdtVolumeSize"
                      },
                      "\n",
                      "OstVolumeSize: ",
                      {
                        "Ref" : "OstVolumeSize"
                      },
                      "\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              },
              "/etc/cloud.env" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "export FSNAME=",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "export PERSONALITY=CLIENT\n",
                      "export LUSTREMD=",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              },
              "/etc/profile.d/openmpi.sh" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "module add openmpi-x86_64\n"
                    ]
                  ]
                },
                "mode" : "0644",
                "owner" : "root",
                "group" : "root"
              }
            }
          }
        }
      },
      "Properties" : {
        "SecurityGroups" : [
          {
            "Ref" : "LustreSecurityGroup"
          }
        ],
        "KeyName" : {
          "Ref" : "KeyName"
        },
        "IamInstanceProfile" : {
          "Ref" : "ClientInstanceProfile"
        },
        "ImageId" : {
          "Fn::FindInMap" : [
            "AWSRegionArch2ServerAMI",
            {
              "Ref" : "AWS::Region"
            },
            {
              "Fn::FindInMap" : [
                "AWSInstanceType2Arch",
                {
                  "Ref" : "ClientInstanceType"
                },
                "Arch"
              ]
            }
          ]
        },
        "InstanceType" : {
          "Ref" : "ClientInstanceType"
        },
        "SpotPrice" : {
          "Fn::FindInMap" : [
            "InstanceSpotPrice",
            {
              "Ref" : "ClientInstanceType"
            },
            "price"
          ]
        },
        "InstanceMonitoring" : "false",
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : [
              "",
              [
                "#!/bin/bash -ex\n",
                "cfn-init -s ",
                {
                  "Ref" : "AWS::StackName"
                },
                " -r ClientLaunchConfig ",
                "   --region ",
                {
                  "Ref" : "AWS::Region"
                },
                "\n",
                "/root/setup-lustre\n"
              ]
            ]
          }
        }
      }
    },
    "PrivateSubnet" : {
      "Type" : "AWS::EC2::Subnet",
      "Properties" : {
        "VpcId" : {
          "Ref" : "VpcId"
        },
        "CidrBlock" : {
          "Ref" : "VpcPrivateCIDR"
        },
        "Tags" : [
          {
            "Key" : "Application",
            "Value" : {
              "Ref" : "AWS::StackName"
            }
          },
          {
            "Key" : "Network",
            "Value" : "Private"
          }
        ],
        "AvailabilityZone" : {
          "Fn::GetAtt" : [
            "NATDevice",
            "AvailabilityZone"
          ]
        }
      }
    },
    "PrivateRouteTable" : {
      "Type" : "AWS::EC2::RouteTable",
      "Properties" : {
        "VpcId" : {
          "Ref" : "VpcId"
        },
        "Tags" : [
          {
            "Key" : "Application",
            "Value" : {
              "Ref" : "AWS::StackName"
            }
          },
          {
            "Key" : "Network",
            "Value" : "Private"
          }
        ]
      }
    },
    "PrivateRouteGateway" : {
      "Type" : "AWS::EC2::Route",
      "Properties" : {
        "RouteTableId" : {
          "Ref" : "PrivateRouteTable"
        },
        "DestinationCidrBlock" : "0.0.0.0/0",
        "InstanceId" : {
          "Ref" : "NATDevice"
        }
      }
    },
    "PrivateSubnetRouteTableAssociation" : {
      "Type" : "AWS::EC2::SubnetRouteTableAssociation",
      "Properties" : {
        "SubnetId" : {
          "Ref" : "PrivateSubnet"
        },
        "RouteTableId" : {
          "Ref" : "PrivateRouteTable"
        }
      }
    },
    "PrivateNetworkAcl" : {
      "Type" : "AWS::EC2::NetworkAcl",
      "Properties" : {
        "VpcId" : {
          "Ref" : "VpcId"
        },
        "Tags" : [
          {
            "Key" : "Application",
            "Value" : {
              "Ref" : "AWS::StackName"
            }
          },
          {
            "Key" : "Network",
            "Value" : "Private"
          }
        ]
      }
    },
    "InboundPrivateNetworkAclEntry" : {
      "Type" : "AWS::EC2::NetworkAclEntry",
      "Properties" : {
        "NetworkAclId" : {
          "Ref" : "PrivateNetworkAcl"
        },
        "RuleNumber" : "100",
        "Protocol" : "6",
        "RuleAction" : "allow",
        "Egress" : "false",
        "CidrBlock" : "0.0.0.0/0",
        "PortRange" : {
          "From" : "0",
          "To" : "65535"
        }
      }
    },
    "OutBoundPrivateNetworkAclEntry" : {
      "Type" : "AWS::EC2::NetworkAclEntry",
      "Properties" : {
        "NetworkAclId" : {
          "Ref" : "PrivateNetworkAcl"
        },
        "RuleNumber" : "100",
        "Protocol" : "6",
        "RuleAction" : "allow",
        "Egress" : "true",
        "CidrBlock" : "0.0.0.0/0",
        "PortRange" : {
          "From" : "0",
          "To" : "65535"
        }
      }
    },
    "PrivateSubnetNetworkAclAssociation" : {
      "Type" : "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties" : {
        "SubnetId" : {
          "Ref" : "PrivateSubnet"
        },
        "NetworkAclId" : {
          "Ref" : "PrivateNetworkAcl"
        }
      }
    },
    "NATDevice" : {
      "Type" : "AWS::EC2::Instance",
      "Properties" : {
        "InstanceType" : {
          "Ref" : "NATInstanceType"
        },
        "KeyName" : {
          "Ref" : "KeyName"
        },
        "ImageId" : {
          "Fn::FindInMap" : [
            "AWSNATAMI",
            {
              "Ref" : "AWS::Region"
            },
            "AMI"
          ]
        },
        "SourceDestCheck" : "false",
        "NetworkInterfaces" : [
          {
            "GroupSet" : [
              {
                "Ref" : "NATSecurityGroup"
              }
            ],
            "AssociatePublicIpAddress" : "true",
            "DeviceIndex" : "0",
            "DeleteOnTermination" : "true",
            "SubnetId" : {
              "Ref" : "VpcPublicSubnetId"
            }
          }
        ],
        "Tags" : [
          {
            "Key" : "Application",
            "Value" : {
              "Ref" : "AWS::StackId"
            }
          },
          {
            "Key" : "Network",
            "Value" : "Public"
          },
          {
            "Key" : "Name",
            "Value" : "NATDevice"
          }
        ]
      }
    },
    "NATSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Enable internal access to the NAT device",
        "VpcId" : {
          "Ref" : "VpcId"
        },
        "SecurityGroupIngress" : [
          {
            "IpProtocol" : "tcp",
            "FromPort" : "80",
            "ToPort" : "80",
            "CidrIp" : {
              "Ref" : "VpcPrivateCIDR"
            }
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "443",
            "ToPort" : "443",
            "CidrIp" : {
              "Ref" : "VpcPrivateCIDR"
            }
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "22",
            "ToPort" : "22",
            "CidrIp" : {
              "Ref" : "SSHFrom"
            }
          }
        ],
        "SecurityGroupEgress" : [
          {
            "IpProtocol" : "tcp",
            "FromPort" : "80",
            "ToPort" : "80",
            "CidrIp" : "0.0.0.0/0"
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "443",
            "ToPort" : "443",
            "CidrIp" : "0.0.0.0/0"
          }
        ]
      }
    },
    "LustreSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Subnet access to Lustre servers",
        "SecurityGroupIngress" : [
          {
            "IpProtocol" : "tcp",
            "FromPort" : "22",
            "ToPort" : "22",
            "CidrIp" : {
              "Ref" : "AccessFrom"
            }
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "988",
            "ToPort" : "988",
            "CidrIp" : {
              "Ref" : "AccessFrom"
            }
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "80",
            "ToPort" : "80",
            "CidrIp" : {
              "Ref" : "AccessFrom"
            }
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "80",
            "ToPort" : "80",
            "CidrIp" : {
              "Ref" : "HTTPFrom"
            }
          },
          {
            "IpProtocol" : "tcp",
            "FromPort" : "22",
            "ToPort" : "22",
            "CidrIp" : {
              "Ref" : "SSHFrom"
            }
          }
        ],
        "VpcId" : {
          "Ref" : "VpcId"
        }
      }
    },
    "LustreInternalSecurityGroupIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "IpProtocol" : "-1",
        "FromPort" : "0",
        "ToPort" : "65535",
        "SourceSecurityGroupId" : {
          "Ref" : "LustreSecurityGroup"
        },
        "GroupId" : {
          "Ref" : "LustreSecurityGroup"
        }
      }
    },
    "MGSNode" : {
      "Properties" : {
        "KeyName" : {
          "Ref" : "KeyName"
        },
        "IamInstanceProfile" : {
          "Ref" : "ServerInstanceProfile"
        },
        "ImageId" : {
          "Fn::FindInMap" : [
            "AWSRegionArch2ServerAMI",
            {
              "Ref" : "AWS::Region"
            },
            {
              "Fn::FindInMap" : [
                "AWSInstanceType2Arch",
                {
                  "Ref" : "MgsInstanceType"
                },
                "Arch"
              ]
            }
          ]
        },
        "InstanceType" : {
          "Ref" : "MgsInstanceType"
        },
        "EbsOptimized" : "true",
        "Tags" : [
          {
            "Key" : "lustre:type",
            "Value" : "mgt",
            "PropagateAtLaunch" : "true"
          },
          {
            "Key" : "loci:application",
            "Value" : { "Ref" : "AWS::StackId"},
            "PropagateAtLaunch" : "true"
          }

        ],
        "UserData" : {
          "Fn::Base64" : {
            "Fn::Join" : [
              "",
              [
                "#!/bin/bash -ex\n",
                "cfn-init -s ",
                {
                  "Ref" : "AWS::StackName"
                },
                " -r MGSNode",
                "   --region ",
                {
                  "Ref" : "AWS::Region"
                },
                "\n",
                "/root/setup-lustre\n"
              ]
            ]
          }
        },
        "BlockDeviceMappings" : [
          {
            "DeviceName" : "/dev/sdf",
            "Ebs" : {
              "VolumeSize" : {
                "Ref" : "MgtVolumeSize"
              }
            }
          }
        ],
        "NetworkInterfaces" : [
          {
            "GroupSet" : [
              {
                "Ref" : "LustreSecurityGroup"
              }
            ],
            "AssociatePublicIpAddress" : "true",
            "DeviceIndex" : "0",
            "DeleteOnTermination" : "true",
            "SubnetId" : {
              "Ref" : "VpcPublicSubnetId"
            }
          }
        ]
      },
      "Type" : "AWS::EC2::Instance",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config" : {
            "files" : {
              "/etc/loci.conf" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "[default]\n",
                      "FsName: ",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "HA: true\n",
                      "FilesystemTable: ",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "RolesTable: ",
                      {
                        "Ref" : "lustremdRoles"
                      },
                      "\n",
                      "MdsCount: ",
                      {
                        "Ref" : "MdsCount"
                      },
                      "\n",
                      "OssCount: ",
                      {
                        "Ref" : "OssCount"
                      },
                      "\n",
                      "LustreAZ: ",
                      {
                        "Fn::GetAtt" : [
                          "PrivateSubnet",
                          "AvailabilityZone"
                        ]
                      },
                      "\n",
                      "MgtVolumeSize: ",
                      {
                        "Ref" : "MgtVolumeSize"
                      },
                      "\n",
                      "MdtVolumeSize: ",
                      {
                        "Ref" : "MdtVolumeSize"
                      },
                      "\n",
                      "OstVolumeSize: ",
                      {
                        "Ref" : "OstVolumeSize"
                      },
                      "\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              },
              "/etc/cloud.env" : {
                "content" : {
                  "Fn::Join" : [
                    "",
                    [
                      "export FSNAME=",
                      {
                        "Ref" : "FsName"
                      },
                      "\n",
                      "export PERSONALITY=MGS\n",
                      "export USING_EBS=true\n",
                      "export LUSTREMD=",
                      {
                        "Ref" : "lustremd"
                      },
                      "\n",
                      "export HA=false\n"
                    ]
                  ]
                },
                "user" : "root",
                "group" : "root",
                "mode" : "00700"
              }
            }
          }
        }
      }
    }
  },
  "Outputs" : {
    "Dashboard" : {
      "Description" : "Monitoring interface for the Lustre servers.",
      "Value" : {
        "Fn::Join" : [
          "",
          [
            "http://",
            {
              "Fn::GetAtt" : [
                "MGSNode",
                "PublicDnsName"
              ]
            },
            "/ganglia"
          ]
        ]
      }
    },
    "LustreExport" : {
      "Description" : "Mount target for Lustre file system.",
      "Value" : {
        "Fn::Join" : [
          "",
          [
            {
              "Fn::GetAtt" : [
                "MGSNode",
                "PrivateIp"
              ]
            },
            ":/",
            {
              "Ref" : "FsName"
            }
          ]
        ]
      }
    }
  }
}