# Observing condition decision tool: monitor conditions and plan HETDEX
# observations
# Copyright (C) 2017, 2018 "The HETDEX collaboration"
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
'''Subcommand to start, stop and manage a docker container running a mysql
server'''
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import argparse as ap
import sys
import time
import pymysql
import ocd.config as ocd_config
from ocd import shots_db
from ocd import utils
try:
import docker
except ImportError: # pragma: no cover
docker = None
def common_parser():
parser = ap.ArgumentParser(add_help=False)
title = 'Override options in the [docker_mysql] section'
overrides_dm = parser.add_argument_group(title=title)
overrides_dm.add_argument('-c', '--container-name',
dest='setting__docker_mysql__container_name',
metavar='CONTAINER_NAME',
help='''Name of the docker container running the
mysql server''')
return parser
[docs]def docker_subcommand(subparsers):
'''Add a subcommand "docker_mysql" to handle the configuration file.
Parameters
----------
subparsers : argparse subparsers object
subparser to use to generate new parsers
Returns
-------
parser : :class:`argparse.ArgumentParser`
modified parser
'''
description = '''OCD subcommand that manages a docker container used to
run a mysql server useful for test OCD.'''
if docker is None:
description += ''' This command is available only if the docker package
is installed. You can get it installing OCD as: ``pip
install ocd[docker]``'''
parser = subparsers.add_parser('docker_mysql',
help='Run MySQL in a docker container',
description=description)
# If ``docker`` is not installed, create the command, but do nothing about
# it
if docker is not None:
subsubparser = parser.add_subparsers(title='''Docker-mysql
subcommands''',
description="""Type '%(prog)s cmd
-h' for detailed information about
the subcommands""",
dest='docker_subcmd')
subsubparser.required = True
subsubparser = docker_up_parser(subsubparser)
subsubparser = docker_down_parser(subsubparser)
subsubparser = docker_info_parser(subsubparser)
return subparsers
[docs]def docker_up_parser(subsubparser):
'''Add the ``docker_mysql up`` parser.
Parameters
----------
subsubparsers : argparse subparsers object
subparser to use to generate new parsers
Returns
-------
subsubparsers : argparse subparsers object
modified subparser
'''
up = subsubparser.add_parser('up',
help='''Start the mysql docker container''',
description='''Start the docker container,
add the ``vl_obsnum`` table and load some
test data''', parents=[common_parser(), ],
formatter_class=ap.ArgumentDefaultsHelpFormatter)
up.set_defaults(func=docker_up)
up = utils.override_epilog_msg(up)
up = ocd_config.config_file_argument(up)
up.add_argument('-r', '--remove-on-error', action='store_true',
help='''If something non expected happens when making
contact with the database, remove the running container''')
return subsubparser
[docs]def docker_down_parser(subsubparser):
'''Add the ``docker_mysql up`` parser.
Parameters
----------
subsubparsers : argparse subparsers object
subparser to use to generate new parsers
Returns
-------
subsubparsers : argparse subparsers object
modified subparser
'''
down = subsubparser.add_parser('down', help='''Stop and remove the mysql
docker container''',
description='''Stop the docker container and
remove it, together with the associated
volume''', parents=[common_parser(), ])
down.set_defaults(func=docker_down)
down = utils.override_epilog_msg(down)
down = ocd_config.config_file_argument(down)
down.add_argument('--no-volumes', action='store_false', help='''Do not
remove the volume associated with the container. Use it
with caution, to avoid filling your disk with temporary
data''')
return subsubparser
[docs]def docker_info_parser(subsubparser):
'''Add the ``docker_mysql up`` parser.
Parameters
----------
subsubparsers : argparse subparsers object
subparser to use to generate new parsers
Returns
-------
subsubparsers : argparse subparsers object
modified subparser
'''
info = subsubparser.add_parser('info', help='''Get information about the
docker image''',
description='''Get the status of the docker
container, its IP address and port''',
parents=[common_parser(), ])
info.set_defaults(func=docker_info)
info = utils.override_epilog_msg(info)
info = ocd_config.config_file_argument(info)
return subsubparser
[docs]def docker_up(args):
'''Function implementing the ``ocd docker_mysql up`` command
Parameters
----------
args : :class:`~argparse.Namespace`
parsed command line arguments
'''
# load the configuration file
config = ocd_config.load_config(config_file=args.config, args=args)
# start the docker container
container = _docker_run(config)
# print the infos about the database
_print_container_info(container)
# get the ip and port to use to contact it
ip, port = container_ip_port(container)
config['database']['mysql_host'] = ip
config['database']['mysql_port'] = port
is_connected = wait_for_connection(config)
if not is_connected:
msg = ('No connection could be established with the mysql server in'
' the container {c}. The table {t} could not be created and'
' filled.'.format(c=container.name,
t=shots_db.OBSNUM_TABLE_NAME))
if args.remove_on_error:
_docker_stop_rm(container.name, v=True)
msg += (' The container has been removed')
else:
msg += (' You can use the ``ocd docker_mysql info`` to check'
' if it is a temporary problem and/or ``ocd docker_mysql'
' down`` to remove the container before trying to create'
' it')
sys.exit(msg)
else:
sys.stdout.write('Create table {t} database {db}:'
' '.format(t=shots_db.OBSNUM_TABLE_NAME,
db=config['database']['mysql_database']))
sys.stdout.flush()
shots_db.create_obsnum_table(config)
print('done')
sys.stdout.write('Add a row to table {t}:'
' '.format(t=shots_db.OBSNUM_TABLE_NAME))
sys.stdout.flush()
shots_db.fill_obsnum_table(config)
print('done')
print('The MySQL database is ready to be used with the rest of OCD.')
print('Make sure that the "mysql_host" and "mysql_port" values of the'
' "[database]" configuration entries are set to the IP and port'
' printed above.')
[docs]def docker_down(args):
'''Stop and remove the mysql docker container
Parameters
----------
args : :class:`~argparse.Namespace`
parsed command line arguments
'''
# load the configuration file
config = ocd_config.load_config(config_file=args.config, args=args)
container_name = config['docker_mysql']['container_name']
# get the docker container, stop it and then remove it
try:
_docker_stop_rm(container_name, args.no_volumes)
except docker.errors.NotFound as e:
sys.exit(e.explanation + '\nNo container to remove.')
[docs]def docker_info(args):
'''Stop and remove the mysql docker container
Parameters
----------
args : :class:`~argparse.Namespace`
parsed command line arguments
'''
# load the configuration file
config = ocd_config.load_config(config_file=args.config, args=args)
container_name = config['docker_mysql']['container_name']
try:
container = get_container(container_name)
except docker.errors.NotFound as e:
msg = e.explanation + '\n'
msg += 'Run ``ocd docker_mysql up`` to start the container'
sys.exit(msg)
_print_container_info(container)
[docs]def container_ip_port(container):
'''Get the IP of the container and the port that it exposes. They can be
used to connect to the mysql server.
.. note::
The port is 3306
Parameters
----------
container : :class:`docker.models.containers.Container`
started container
Returns
-------
ip, port : string
IP address allocated for the container and the exposed port
'''
return container.attrs['NetworkSettings']['IPAddress'], '3306'
[docs]def get_container(container_name):
'''Get the container running the mysql server
Parameters
----------
container_name : string
name of the container to get
Returns
-------
:class:`docker.models.containers.Container`
started container
'''
client = docker.from_env(version='auto')
return client.containers.get(container_name)
[docs]def wait_for_connection(conf, max_attempts=20, sleep=1):
'''Wait until a connection with the mysql server can be made.
Any exception of type :exc:`pymysql.err.OperationalError` with error code
different from ``2003`` is re-raised
Parameters
----------
conf : :class:`pyhetdex.tools.configuration.ConfigParser`
configuration
max_attempts : int, optional
maximum number of attempts before giving up
sleep : float, optional
time to wait before attempting again the connection
Returns
-------
bool
``True`` if the connection happens, ``False`` otherwise
'''
sys.stdout.write('Connect to the MySQL server')
sys.stdout.flush()
for i in range(max_attempts):
try:
# try to connect for 10 times, after that give up
with shots_db.mysql_connection(conf):
print(' Connected')
return True
except pymysql.err.OperationalError as e:
if e.args[0] == 2003:
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(sleep)
else:
raise
else:
print('Failed')
return False
[docs]def _docker_run(conf):
'''Run the mysql docker image
Parameters
----------
client : :class:`docker.client.DockerClient`
docker client
conf : :class:`pyhetdex.tools.configuration.ConfigParser`
configuration
Returns
-------
container : :class:`docker.models.containers.Container`
started container
'''
# create the docker client
client = docker.from_env(version='auto')
# set the environment variables
conf_db = conf['database']
environment = dict(MYSQL_ROOT_PASSWORD='test',
MYSQL_DATABASE=conf_db['mysql_database'],
MYSQL_USER=conf_db['mysql_user'],
MYSQL_PASSWORD=conf_db['mysql_password'])
# run the mysql docker image
db_name = conf['docker_mysql']['container_name']
container = client.containers.run('mysql:latest', name=db_name,
environment=environment, detach=True)
container.reload() # update the status and attrs
return container
[docs]def _docker_stop_rm(container_name, v=True):
'''Stop and remove mysql docker the container.
Parameters
----------
container_name : string
name of the container to get
v : bool, optional
remove the volumes associated with the container
Returns
-------
container : :class:`docker.models.containers.Container`
started container
'''
container = get_container(container_name)
container.stop()
container.reload()
print('The docker container {c.name} status is:'
' {c.status}'.format(c=container))
container.remove(v=v)
print('The docker container {c.name} has been removed'.format(c=container))
[docs]def _print_container_info(container):
'''Print information about the container
Parameters
----------
container : :class:`docker.models.containers.Container`
started container
'''
print('The docker container {c.name} status is:'
' {c.status}'.format(c=container))
if container.status == 'running':
print('The mysql server can be accessed at'
' {}:{}.'.format(*container_ip_port(container)))
else:
print('(Re)start the container to have the IP and port')