packstack_ironic_ocata/packstack/plugins/swift_600.py

# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Installs and configures Swift
"""

import re
import uuid
import netaddr

from packstack.installer import basedefs
from packstack.installer import validators
from packstack.installer import processors
from packstack.installer.exceptions import ParamValidationError
from packstack.installer import utils
from packstack.installer.utils import split_hosts

from packstack.modules.documentation import update_params_usage

# ------------- Swift Packstack Plugin Initialization --------------

PLUGIN_NAME = "OS-Swift"
PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')


def initConfig(controller):
    params = [
        {"CMD_OPTION": "os-swift-ks-passwd",
         "PROMPT": "Enter the password for the Swift Keystone access",
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_not_empty],
         "DEFAULT_VALUE": "PW_PLACEHOLDER",
         "PROCESSORS": [processors.process_password],
         "MASK_INPUT": True,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_SWIFT_KS_PW",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": True,
         "CONDITION": False},

        {"CMD_OPTION": "os-swift-storages",
         "PROMPT": "Enter the Swift Storage devices e.g. /path/to/dev",
         "OPTION_LIST": [],
         "VALIDATORS": [validate_storage],
         "DEFAULT_VALUE": '',
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_SWIFT_STORAGES",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False,
         "DEPRECATES": ['CONFIG_SWIFT_STORAGE_HOSTS']},

        {"CMD_OPTION": "os-swift-storage-zones",
         "PROMPT": ("Enter the number of swift storage zones, MUST be no "
                    "bigger than the number of storage devices configured"),
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_integer],
         "DEFAULT_VALUE": "1",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_SWIFT_STORAGE_ZONES",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},

        {"CMD_OPTION": "os-swift-storage-replicas",
         "PROMPT": ("Enter the number of swift storage replicas, MUST be no "
                    "bigger than the number of storage zones configured"),
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_integer],
         "DEFAULT_VALUE": "1",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_SWIFT_STORAGE_REPLICAS",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},

        {"CMD_OPTION": "os-swift-storage-fstype",
         "PROMPT": "Enter FileSystem type for storage nodes",
         "OPTION_LIST": ['xfs', 'ext4'],
         "VALIDATORS": [validators.validate_options],
         "DEFAULT_VALUE": "ext4",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_SWIFT_STORAGE_FSTYPE",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},

        {"CMD_OPTION": "os-swift-hash",
         "PROMPT": "Enter hash for Swift shared secret",
         "OPTION_LIST": [],
         "VALIDATORS": [validators.validate_not_empty],
         "DEFAULT_VALUE": uuid.uuid4().hex[:16],
         "MASK_INPUT": True,
         "LOOSE_VALIDATION": False,
         "CONF_NAME": "CONFIG_SWIFT_HASH",
         "USE_DEFAULT": True,
         "NEED_CONFIRM": True,
         "CONDITION": False},

        {"CMD_OPTION": "os-swift-storage-size",
         "PROMPT": ("Enter the size of the storage device (eg. 2G, 2000M, "
                    "2000000K)"),
         "OPTION_LIST": [],
         "VALIDATORS": [validate_storage_size],
         "DEFAULT_VALUE": "2G",
         "MASK_INPUT": False,
         "LOOSE_VALIDATION": True,
         "CONF_NAME": "CONFIG_SWIFT_STORAGE_SIZE",
         "USE_DEFAULT": False,
         "NEED_CONFIRM": False,
         "CONDITION": False},
    ]
    update_params_usage(basedefs.PACKSTACK_DOC, params, sectioned=False)
    group = {"GROUP_NAME": "OSSWIFT",
             "DESCRIPTION": "OpenStack Swift Config parameters",
             "PRE_CONDITION": "CONFIG_SWIFT_INSTALL",
             "PRE_CONDITION_MATCH": "y",
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, params)


def initSequences(controller):
    if controller.CONF['CONFIG_SWIFT_INSTALL'] != 'y':
        return

    steps = [
        {'title': 'Preparing Swift builder entries',
         'functions': [create_builder_manifest]},
        {'title': 'Preparing Swift proxy entries',
         'functions': [create_proxy_manifest]},
        {'title': 'Preparing Swift storage entries',
         'functions': [create_storage_manifest]},
    ]
    controller.addSequence("Installing OpenStack Swift", [], [], steps)


# ------------------------- helper functions -------------------------

def validate_storage(param, options=None):
    if not param:
        return
    if not param.startswith('/'):
        raise ParamValidationError(
            'Storage value has to be in format "/path/to/device".'
        )


def validate_storage_size(param, options=None):
    match = re.match(r'\d+G|\d+M|\d+K', param, re.IGNORECASE)
    if not match:
        msg = 'Storage size not have a valid value (eg. 1G, 1000M, 1000000K)'
        raise ParamValidationError(msg)


def parse_devices(config):
    """
    Returns dict containing information about Swift storage devices.
    """
    devices = []
    device_number = 0
    num_zones = int(config["CONFIG_SWIFT_STORAGE_ZONES"])
    for device in config["CONFIG_SWIFT_STORAGES"].split(","):
        # we have to get rid of host part in case deprecated parameter
        # CONFIG_SWIFT_STORAGE_HOSTS has been used
        if ':' in device:
            device = device.split(':')[1]
        # device should be empty string in case only IP address has been used
        try:
            netaddr.IPAddress(device)
        except Exception:
            device = device.strip()
        else:
            device = ''

        if not device:
            continue
        device_number += 1
        zone = str((device_number % num_zones) + 1)
        devices.append({'device': device, 'zone': zone,
                        'device_name': 'device%s' % device_number})
    if not devices:
        devices.append({'device': None, 'zone': 1,
                        'device_name': 'swiftloopback'})
        config['CONFIG_SWIFT_LOOPBACK'] = 'y'
    else:
        config['CONFIG_SWIFT_LOOPBACK'] = 'n'
    return devices


def check_device(host, device):
    """
    Raises ScriptRuntimeError if given device is not mounted on given
    host.
    """
    server = utils.ScriptRunner(host)

    # the device MUST exist
    cmd = 'ls -l %s'
    server.append(cmd % device)

    # if it is not mounted then we can use it
    cmd = 'grep "%s " /proc/self/mounts || exit 0'
    server.append(cmd % device)

    # if it is mounted then the mount point has to be in /srv/node
    cmd = 'grep "%s /srv/node" /proc/self/mounts && exit 0'
    server.append(cmd % device)

    # if we got here without exiting then we can't use this device
    server.append('exit 1')
    server.execute()


def get_storage_size(config):
    ranges = {'G': 1048576, 'M': 1024, 'K': 1}
    size = config['CONFIG_SWIFT_STORAGE_SIZE'].strip()
    for measure in ['G', 'M', 'K']:
        if re.match('\d+' + measure, size, re.IGNORECASE):
            intsize = int(size.rstrip(measure)) * ranges[measure]
            return intsize


# -------------------------- step functions --------------------------


def create_builder_manifest(config, messages):
    global devices
    devices = parse_devices(config)
    # The ring file should be built and distributed before the storage services
    # come up. Specifically the replicator crashes if the ring isn't present

    def device_def(dev_type, host, dev_port, devicename, zone):
        # device host has to be IP address
        host = utils.force_ip(host)
        fmt = ('\n@@%s { "%s:%s/%s":\n'
               '  zone   => %s,\n'
               '  weight => 10, }\n')
        return fmt % (dev_type, host, dev_port, devicename, zone)

    # Add each device to the ring
    devicename = 0
    for configkey, dev_type, dev_port in (
        [('SWIFT_RING_OBJECT_DEVICES', 'ring_object_device', 6000),
         ('SWIFT_RING_CONTAINER_DEVICES', 'ring_container_device', 6001),
         ('SWIFT_RING_ACCOUNT_DEVICES', 'ring_account_device', 6002)]):
        swift_dev_details = dict()
        host = utils.force_ip(config['CONFIG_STORAGE_HOST_URL'])
        fstype = config["CONFIG_SWIFT_STORAGE_FSTYPE"]
        for device in devices:
            devicename = device['device_name']
            key = "dev_%s_%s" % (host, devicename)
            swift_dev_details.setdefault(key, {})
            zone = device['zone']
            swift_dev_details[key]['name'] = "%s:%s/%s" % (host, dev_port,
                                                           devicename)
            swift_dev_details[key]['weight'] = "%s" % 10
            swift_dev_details[key]['zone'] = "%s" % zone
        config[configkey] = swift_dev_details


def create_proxy_manifest(config, messages):
    fw_details = dict()
    key = "swift_proxy"
    fw_details.setdefault(key, {})
    fw_details[key]['host'] = "ALL"
    fw_details[key]['service_name'] = "swift proxy"
    fw_details[key]['chain'] = "INPUT"
    fw_details[key]['ports'] = ['8080']
    fw_details[key]['proto'] = "tcp"
    config['FIREWALL_SWIFT_PROXY_RULES'] = fw_details


def create_storage_manifest(config, messages):
    global devices

    devicename = 0
    swift_dev_details = dict()
    host = utils.force_ip(config['CONFIG_STORAGE_HOST_URL'])
    fstype = config["CONFIG_SWIFT_STORAGE_FSTYPE"]

    # this need to happen once per storage device
    for device in devices:
        if device['device'] is None:
            config['CONFIG_SWIFT_STORAGE_SEEK'] = get_storage_size(config)
        else:
            devicename = device['device_name']
            devicedev = device['device']
            key = "dev_%s_%s" % (host, devicename)
            swift_dev_details.setdefault(key, {})
            swift_dev_details[key]['device'] = "%s" % devicename
            swift_dev_details[key]['dev'] = "%s" % devicedev
            swift_dev_details[key]['fstype'] = "%s" % fstype
    config['CONFIG_SWIFT_STORAGE_DEVICES'] = swift_dev_details

    # set allowed hosts for firewall
    hosts = set([config['CONFIG_STORAGE_HOST']])
    if config['CONFIG_NOVA_INSTALL'] == 'y':
        hosts |= split_hosts(config['CONFIG_COMPUTE_HOSTS'])

    fw_details = dict()
    for host in hosts:
        key = "swift_storage_and_rsync_%s" % host
        fw_details.setdefault(key, {})
        fw_details[key]['host'] = "%s" % host
        fw_details[key]['service_name'] = "swift storage and rsync"
        fw_details[key]['chain'] = "INPUT"
        fw_details[key]['ports'] = ['6000', '6001', '6002', '873']
        fw_details[key]['proto'] = "tcp"
    config['FIREWALL_SWIFT_STORAGE_RULES'] = fw_details