# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Installs and configures Swift
"""
import re
import uuid
import netaddr
from packstack.installer import basedefs
from packstack.installer import validators
from packstack.installer import processors
from packstack.installer.exceptions import ParamValidationError
from packstack.installer import utils
from packstack.installer.utils import split_hosts
from packstack.modules.documentation import update_params_usage
# ------------- Swift Packstack Plugin Initialization --------------
PLUGIN_NAME = "OS-Swift"
PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')
def initConfig(controller):
params = [
{"CMD_OPTION": "os-swift-ks-passwd",
"PROMPT": "Enter the password for the Swift Keystone access",
"OPTION_LIST": [],
"VALIDATORS": [validators.validate_not_empty],
"DEFAULT_VALUE": "PW_PLACEHOLDER",
"PROCESSORS": [processors.process_password],
"MASK_INPUT": True,
"LOOSE_VALIDATION": False,
"CONF_NAME": "CONFIG_SWIFT_KS_PW",
"USE_DEFAULT": False,
"NEED_CONFIRM": True,
"CONDITION": False},
{"CMD_OPTION": "os-swift-storages",
"PROMPT": "Enter the Swift Storage devices e.g. /path/to/dev",
"OPTION_LIST": [],
"VALIDATORS": [validate_storage],
"DEFAULT_VALUE": '',
"MASK_INPUT": False,
"LOOSE_VALIDATION": True,
"CONF_NAME": "CONFIG_SWIFT_STORAGES",
"USE_DEFAULT": False,
"NEED_CONFIRM": False,
"CONDITION": False,
"DEPRECATES": ['CONFIG_SWIFT_STORAGE_HOSTS']},
{"CMD_OPTION": "os-swift-storage-zones",
"PROMPT": ("Enter the number of swift storage zones, MUST be no "
"bigger than the number of storage devices configured"),
"OPTION_LIST": [],
"VALIDATORS": [validators.validate_integer],
"DEFAULT_VALUE": "1",
"MASK_INPUT": False,
"LOOSE_VALIDATION": True,
"CONF_NAME": "CONFIG_SWIFT_STORAGE_ZONES",
"USE_DEFAULT": False,
"NEED_CONFIRM": False,
"CONDITION": False},
{"CMD_OPTION": "os-swift-storage-replicas",
"PROMPT": ("Enter the number of swift storage replicas, MUST be no "
"bigger than the number of storage zones configured"),
"OPTION_LIST": [],
"VALIDATORS": [validators.validate_integer],
"DEFAULT_VALUE": "1",
"MASK_INPUT": False,
"LOOSE_VALIDATION": True,
"CONF_NAME": "CONFIG_SWIFT_STORAGE_REPLICAS",
"USE_DEFAULT": False,
"NEED_CONFIRM": False,
"CONDITION": False},
{"CMD_OPTION": "os-swift-storage-fstype",
"PROMPT": "Enter FileSystem type for storage nodes",
"OPTION_LIST": ['xfs', 'ext4'],
"VALIDATORS": [validators.validate_options],
"DEFAULT_VALUE": "ext4",
"MASK_INPUT": False,
"LOOSE_VALIDATION": True,
"CONF_NAME": "CONFIG_SWIFT_STORAGE_FSTYPE",
"USE_DEFAULT": False,
"NEED_CONFIRM": False,
"CONDITION": False},
{"CMD_OPTION": "os-swift-hash",
"PROMPT": "Enter hash for Swift shared secret",
"OPTION_LIST": [],
"VALIDATORS": [validators.validate_not_empty],
"DEFAULT_VALUE": uuid.uuid4().hex[:16],
"MASK_INPUT": True,
"LOOSE_VALIDATION": False,
"CONF_NAME": "CONFIG_SWIFT_HASH",
"USE_DEFAULT": True,
"NEED_CONFIRM": True,
"CONDITION": False},
{"CMD_OPTION": "os-swift-storage-size",
"PROMPT": ("Enter the size of the storage device (eg. 2G, 2000M, "
"2000000K)"),
"OPTION_LIST": [],
"VALIDATORS": [validate_storage_size],
"DEFAULT_VALUE": "2G",
"MASK_INPUT": False,
"LOOSE_VALIDATION": True,
"CONF_NAME": "CONFIG_SWIFT_STORAGE_SIZE",
"USE_DEFAULT": False,
"NEED_CONFIRM": False,
"CONDITION": False},
]
update_params_usage(basedefs.PACKSTACK_DOC, params, sectioned=False)
group = {"GROUP_NAME": "OSSWIFT",
"DESCRIPTION": "OpenStack Swift Config parameters",
"PRE_CONDITION": "CONFIG_SWIFT_INSTALL",
"PRE_CONDITION_MATCH": "y",
"POST_CONDITION": False,
"POST_CONDITION_MATCH": True}
controller.addGroup(group, params)
def initSequences(controller):
if controller.CONF['CONFIG_SWIFT_INSTALL'] != 'y':
return
steps = [
{'title': 'Preparing Swift builder entries',
'functions': [create_builder_manifest]},
{'title': 'Preparing Swift proxy entries',
'functions': [create_proxy_manifest]},
{'title': 'Preparing Swift storage entries',
'functions': [create_storage_manifest]},
]
controller.addSequence("Installing OpenStack Swift", [], [], steps)
# ------------------------- helper functions -------------------------
def validate_storage(param, options=None):
if not param:
return
if not param.startswith('/'):
raise ParamValidationError(
'Storage value has to be in format "/path/to/device".'
)
def validate_storage_size(param, options=None):
match = re.match(r'\d+G|\d+M|\d+K', param, re.IGNORECASE)
if not match:
msg = 'Storage size not have a valid value (eg. 1G, 1000M, 1000000K)'
raise ParamValidationError(msg)
def parse_devices(config):
"""
Returns dict containing information about Swift storage devices.
"""
devices = []
device_number = 0
num_zones = int(config["CONFIG_SWIFT_STORAGE_ZONES"])
for device in config["CONFIG_SWIFT_STORAGES"].split(","):
# we have to get rid of host part in case deprecated parameter
# CONFIG_SWIFT_STORAGE_HOSTS has been used
if ':' in device:
device = device.split(':')[1]
# device should be empty string in case only IP address has been used
try:
netaddr.IPAddress(device)
except Exception:
device = device.strip()
else:
device = ''
if not device:
continue
device_number += 1
zone = str((device_number % num_zones) + 1)
devices.append({'device': device, 'zone': zone,
'device_name': 'device%s' % device_number})
if not devices:
devices.append({'device': None, 'zone': 1,
'device_name': 'swiftloopback'})
config['CONFIG_SWIFT_LOOPBACK'] = 'y'
else:
config['CONFIG_SWIFT_LOOPBACK'] = 'n'
return devices
def check_device(host, device):
"""
Raises ScriptRuntimeError if given device is not mounted on given
host.
"""
server = utils.ScriptRunner(host)
# the device MUST exist
cmd = 'ls -l %s'
server.append(cmd % device)
# if it is not mounted then we can use it
cmd = 'grep "%s " /proc/self/mounts || exit 0'
server.append(cmd % device)
# if it is mounted then the mount point has to be in /srv/node
cmd = 'grep "%s /srv/node" /proc/self/mounts && exit 0'
server.append(cmd % device)
# if we got here without exiting then we can't use this device
server.append('exit 1')
server.execute()
def get_storage_size(config):
ranges = {'G': 1048576, 'M': 1024, 'K': 1}
size = config['CONFIG_SWIFT_STORAGE_SIZE'].strip()
for measure in ['G', 'M', 'K']:
if re.match('\d+' + measure, size, re.IGNORECASE):
intsize = int(size.rstrip(measure)) * ranges[measure]
return intsize
# -------------------------- step functions --------------------------
def create_builder_manifest(config, messages):
global devices
devices = parse_devices(config)
# The ring file should be built and distributed before the storage services
# come up. Specifically the replicator crashes if the ring isn't present
def device_def(dev_type, host, dev_port, devicename, zone):
# device host has to be IP address
host = utils.force_ip(host)
fmt = ('\n@@%s { "%s:%s/%s":\n'
' zone => %s,\n'
' weight => 10, }\n')
return fmt % (dev_type, host, dev_port, devicename, zone)
# Add each device to the ring
devicename = 0
for configkey, dev_type, dev_port in (
[('SWIFT_RING_OBJECT_DEVICES', 'ring_object_device', 6000),
('SWIFT_RING_CONTAINER_DEVICES', 'ring_container_device', 6001),
('SWIFT_RING_ACCOUNT_DEVICES', 'ring_account_device', 6002)]):
swift_dev_details = dict()
host = utils.force_ip(config['CONFIG_STORAGE_HOST_URL'])
fstype = config["CONFIG_SWIFT_STORAGE_FSTYPE"]
for device in devices:
devicename = device['device_name']
key = "dev_%s_%s" % (host, devicename)
swift_dev_details.setdefault(key, {})
zone = device['zone']
swift_dev_details[key]['name'] = "%s:%s/%s" % (host, dev_port,
devicename)
swift_dev_details[key]['weight'] = "%s" % 10
swift_dev_details[key]['zone'] = "%s" % zone
config[configkey] = swift_dev_details
def create_proxy_manifest(config, messages):
fw_details = dict()
key = "swift_proxy"
fw_details.setdefault(key, {})
fw_details[key]['host'] = "ALL"
fw_details[key]['service_name'] = "swift proxy"
fw_details[key]['chain'] = "INPUT"
fw_details[key]['ports'] = ['8080']
fw_details[key]['proto'] = "tcp"
config['FIREWALL_SWIFT_PROXY_RULES'] = fw_details
def create_storage_manifest(config, messages):
global devices
devicename = 0
swift_dev_details = dict()
host = utils.force_ip(config['CONFIG_STORAGE_HOST_URL'])
fstype = config["CONFIG_SWIFT_STORAGE_FSTYPE"]
# this need to happen once per storage device
for device in devices:
if device['device'] is None:
config['CONFIG_SWIFT_STORAGE_SEEK'] = get_storage_size(config)
else:
devicename = device['device_name']
devicedev = device['device']
key = "dev_%s_%s" % (host, devicename)
swift_dev_details.setdefault(key, {})
swift_dev_details[key]['device'] = "%s" % devicename
swift_dev_details[key]['dev'] = "%s" % devicedev
swift_dev_details[key]['fstype'] = "%s" % fstype
config['CONFIG_SWIFT_STORAGE_DEVICES'] = swift_dev_details
# set allowed hosts for firewall
hosts = set([config['CONFIG_STORAGE_HOST']])
if config['CONFIG_NOVA_INSTALL'] == 'y':
hosts |= split_hosts(config['CONFIG_COMPUTE_HOSTS'])
fw_details = dict()
for host in hosts:
key = "swift_storage_and_rsync_%s" % host
fw_details.setdefault(key, {})
fw_details[key]['host'] = "%s" % host
fw_details[key]['service_name'] = "swift storage and rsync"
fw_details[key]['chain'] = "INPUT"
fw_details[key]['ports'] = ['6000', '6001', '6002', '873']
fw_details[key]['proto'] = "tcp"
config['FIREWALL_SWIFT_STORAGE_RULES'] = fw_details