summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Styk <mastyk@redhat.com>2019-05-22 12:59:36 +0200
committerMartin Styk <mastyk@redhat.com>2019-05-22 13:46:36 +0200
commitffeeed41e867fc21a53eb055c9bdfca38f45d5b2 (patch)
tree9e85386ac3d1e2d98a3b2f55c50568a14ff45c71
parente568a899ee3cc624cdf4257b65d0c9ffcdc0da34 (diff)
Use session for OpenStack connection in inttest
Change-Id: Id5496bd91720fdbeaabc1d03407aef7089275e91 Signed-off-by: Martin Styk <mastyk@redhat.com>
-rw-r--r--IntegrationTests/src/bkr/inttest/__init__.py115
-rw-r--r--Server/server.cfg2
-rw-r--r--documentation/admin-guide/openstack.rst2
3 files changed, 77 insertions, 42 deletions
diff --git a/IntegrationTests/src/bkr/inttest/__init__.py b/IntegrationTests/src/bkr/inttest/__init__.py
index d8af976..29d7f51 100644
--- a/IntegrationTests/src/bkr/inttest/__init__.py
+++ b/IntegrationTests/src/bkr/inttest/__init__.py
@@ -1,10 +1,10 @@
-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
import pkg_resources
+
pkg_resources.require('SQLAlchemy >= 0.6')
pkg_resources.require('TurboGears >= 1.1')
@@ -16,27 +16,27 @@ import socket
import threading
import subprocess
import shutil
-import uuid
import tempfile
import re
-from StringIO import StringIO
import logging, logging.config
import signal
import unittest2 as unittest
-import cherrypy
import turbogears
-try:
- import keystoneclient.v3.client
- has_keystoneclient = True
-except ImportError:
- has_keystoneclient = False
+
try:
import glanceclient.v2.client
has_glanceclient = True
except ImportError:
has_glanceclient = False
+
+try:
+ from keystoneauth1.identity import v3
+ from keystoneauth1 import session as os_session
+ has_keystoneauth1 = True
+except ImportError:
+ has_keystoneauth1 = False
+
from turbogears.database import session
-from bkr.server import dynamic_virt
from bkr.server.controllers import Root
from bkr.server.model import OpenStackRegion, ConfigItem, User, LabController, Task, Distro
from bkr.server.util import load_config
@@ -49,12 +49,13 @@ from bkr.inttest.mail_capture import MailCaptureThread
orig_cwd = os.getcwd()
os.chdir('/tmp')
import turbogears.testutil
+
os.chdir(orig_cwd)
CONFIG_FILE = os.environ.get('BEAKER_CONFIG_FILE')
-class DatabaseTestCase(unittest.TestCase):
+class DatabaseTestCase(unittest.TestCase):
"""
Tests which touch the database in any way (session.begin()) should inherit
from this, so that the session is cleaned up at the end of each test. This
@@ -83,33 +84,43 @@ class DatabaseTestCase(unittest.TestCase):
def _clear_captured_mails(self):
mail_capture_thread.clear()
+
# workaround for delayed log formatting in nose
# https://groups.google.com/forum/?fromgroups=#!topic/nose-users/5uZVDfDf1ZI
orig_LogRecord = logging.LogRecord
+
+
class EagerFormattedLogRecord(orig_LogRecord):
def __init__(self, *args, **kwargs):
orig_LogRecord.__init__(self, *args, **kwargs)
if self.args:
self.msg = self.msg % self.args
self.args = None
+
+
logging.LogRecord = EagerFormattedLogRecord
log = logging.getLogger(__name__)
+
def get_server_base():
return os.environ.get('BEAKER_SERVER_BASE_URL',
- 'http://localhost:%s/' % turbogears.config.get('server.socket_port'))
+ 'http://localhost:%s/' % turbogears.config.get('server.socket_port'))
+
def with_transaction(func):
"""
Runs the decorated function inside a transaction. Apply to setUp or other
methods as needed.
"""
+
def _decorated(*args, **kwargs):
with session.begin():
func(*args, **kwargs)
+
return _decorated
+
def fix_beakerd_repodata_perms():
# This is ugly, but I can't come up with anything better...
# Any tests which invoke beakerd code directly will create
@@ -123,6 +134,7 @@ def fix_beakerd_repodata_perms():
repodata = os.path.join(turbogears.config.get('basepath.rpms'), 'repodata')
shutil.rmtree(repodata, ignore_errors=True)
+
def check_listen(port):
"""
Returns True iff any process on the system is listening
@@ -131,12 +143,13 @@ def check_listen(port):
# with newer lsof we could just use -sTCP:LISTEN,
# but RHEL5's lsof is too old so we have to filter for LISTEN state ourselves
output, error = subprocess.Popen(['lsof', '-iTCP:%d' % port],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
for line in output.splitlines():
if '(LISTEN)' in line:
return True
return False
+
class Process(object):
"""
Thin wrapper around subprocess.Popen which supports starting and killing
@@ -144,7 +157,7 @@ class Process(object):
"""
def __init__(self, name, args, env=None, listen_port=None,
- stop_signal=signal.SIGTERM, exec_dir=None):
+ stop_signal=signal.SIGTERM, exec_dir=None):
self.name = name
self.args = args
self.env = env
@@ -199,7 +212,7 @@ class Process(object):
log.warning('%s (pid %d) already dead, not killing', self.name, self.popen.pid)
else:
log.info('Sending signal %r to %s (pid %d)',
- self.stop_signal, self.name, self.popen.pid)
+ self.stop_signal, self.name, self.popen.pid)
os.kill(self.popen.pid, self.stop_signal)
self.popen.wait()
@@ -209,6 +222,7 @@ class Process(object):
def finish_output_capture(self):
return self.communicate_thread.finish_capture()
+
class CommunicateThread(threading.Thread):
"""
Nose has support for capturing stdout during tests, by fiddling with sys.stdout.
@@ -242,16 +256,18 @@ class CommunicateThread(threading.Thread):
del self.captured
return result
+
slapd_config_dir = None
slapd_data_dir = None
+
def setup_slapd():
global slapd_config_dir, slapd_data_dir
slapd_config_dir = tempfile.mkdtemp(prefix='beaker-tests-slapd-config')
slapd_data_dir = tempfile.mkdtemp(prefix='beaker-tests-slapd-data')
log.info('Populating slapd config')
slapadd = subprocess.Popen(['slapadd', '-F', slapd_config_dir, '-n0'],
- stdin=subprocess.PIPE)
+ stdin=subprocess.PIPE)
slapadd.communicate("""
dn: cn=config
objectClass: olcGlobal
@@ -283,21 +299,27 @@ olcAccess: to * by * read
assert slapadd.returncode == 0
log.info('Populating slapd data')
subprocess.check_call(['slapadd', '-F', slapd_config_dir, '-n1', '-l',
- pkg_resources.resource_filename('bkr.inttest', 'ldap-data.ldif')],
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ pkg_resources.resource_filename('bkr.inttest', 'ldap-data.ldif')],
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
def cleanup_slapd():
shutil.rmtree(slapd_data_dir, ignore_errors=True)
shutil.rmtree(slapd_config_dir, ignore_errors=True)
+
mail_capture_thread = MailCaptureThread()
+
def _glance():
- if not has_keystoneclient:
- raise RuntimeError('python-keystoneclient is not installed')
+ # First check if we have all dependencies for runtime
if not has_glanceclient:
raise RuntimeError('python-glanceclient is not installed')
- # We upload the ipxe image to Glance using the same dummy credentials and
+
+ if not has_keystoneauth1:
+ raise RuntimeError('python-keystoneauth1 is not installed')
+
+ # We upload the iPXE image to Glance using the same dummy credentials and
# tenant on whose behalf we will be creating VMs later.
username = os.environ['OPENSTACK_DUMMY_USERNAME']
password = os.environ['OPENSTACK_DUMMY_PASSWORD']
@@ -306,16 +328,21 @@ def _glance():
user_domain_name = os.environ.get('OPENSTACK_DUMMY_USER_DOMAIN_NAME')
project_domain_name = os.environ.get('OPENSTACK_DUMMY_PROJECT_DOMAIN_NAME')
- keystone = keystoneclient.v3.client.Client(
+ auth = v3.Password(
+ auth_url=auth_url,
username=username,
+ user_domain_name=user_domain_name,
password=password,
project_name=project_name,
- user_domain_name=user_domain_name,
project_domain_name=project_domain_name,
- auth_url=auth_url)
- glance_url = keystone.service_catalog.url_for(
- service_type='image', endpoint_type='publicURL')
- return glanceclient.v2.client.Client(glance_url, token=keystone.auth_token)
+ )
+
+ sess = os_session.Session(auth=auth)
+ auth_ref = auth.get_auth_ref(sess)
+ glance_url = auth_ref.service_catalog.url_for(service_type='image', interface='public')
+
+ return glanceclient.v2.client.Client(glance_url, token=auth.get_token(sess))
+
def setup_openstack():
with session.begin():
@@ -325,19 +352,22 @@ def setup_openstack():
# tracking instances back to the test run which created them.
admin_user = User.by_user_name(data_setup.ADMIN_USER)
guest_name_prefix = u'beaker-testsuite-%s-%s-' % (
- datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S'),
- socket.gethostname())
+ datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S'),
+ socket.gethostname())
ConfigItem.by_name(u'guest_name_prefix').set(guest_name_prefix, user=admin_user)
ipxe_image.upload_image(_glance(), visibility=u'private')
+
def cleanup_openstack():
with session.begin():
region = OpenStackRegion.query.one()
log.info('Cleaning up Glance image %s', region.ipxe_image_id)
_glance().images.delete(region.ipxe_image_id)
+
processes = None
+
def edit_file(file, old_text, new_text):
with open(file, 'r') as f:
contents = f.read()
@@ -347,6 +377,7 @@ def edit_file(file, old_text, new_text):
tmp_config.flush()
return tmp_config
+
def start_process(name, env=None):
found = False
for p in processes:
@@ -357,6 +388,7 @@ def start_process(name, env=None):
if found is False:
raise ValueError('%s is not a valid process name')
+
def stop_process(name):
found = False
for p in processes:
@@ -366,6 +398,7 @@ def stop_process(name):
if found is False:
raise ValueError('%s is not a valid process name')
+
def setup_package():
assert os.path.exists(CONFIG_FILE), 'Config file %s must exist' % CONFIG_FILE
load_config(configfile=CONFIG_FILE)
@@ -388,20 +421,21 @@ def setup_package():
u'libxml2-python expect pyOpenSSL'.split())
data_setup.create_task(name=u'/distribution/check-install')
data_setup.create_task(name=u'/distribution/reservesys',
- requires=u'emacs vim-enhanced unifdef sendmail'.split())
+ requires=u'emacs vim-enhanced unifdef sendmail'.split())
data_setup.create_task(name=u'/distribution/utils/dummy')
data_setup.create_task(name=u'/distribution/inventory')
if not Distro.query.count():
# The 'BlueShoeLinux5-5' string appears in many tests, because it's
# the distro name used in complete-job.xml.
- data_setup.create_distro_tree(osmajor=u'BlueShoeLinux5', distro_name=u'BlueShoeLinux5-5')
+ data_setup.create_distro_tree(osmajor=u'BlueShoeLinux5',
+ distro_name=u'BlueShoeLinux5-5')
if os.path.exists(turbogears.config.get('basepath.rpms')):
# Remove any task RPMs left behind by previous test runs
for entry in os.listdir(turbogears.config.get('basepath.rpms')):
shutil.rmtree(os.path.join(
- turbogears.config.get('basepath.rpms'),
- entry), ignore_errors=True)
+ turbogears.config.get('basepath.rpms'),
+ entry), ignore_errors=True)
else:
os.mkdir(turbogears.config.get('basepath.rpms'))
@@ -423,16 +457,16 @@ def setup_package():
# requirements in bkr.server.wsgi
processes.extend([
Process('gunicorn', args=[sys.executable, '-c',
- '__requires__ = ["CherryPy < 3.0"]; import pkg_resources; ' \
- 'from gunicorn.app.wsgiapp import run; run()',
- '--bind', ':%s' % turbogears.config.get('server.socket_port'),
- '--workers', '8', '--access-logfile', '-', '--preload',
- 'bkr.server.wsgi:application'],
- listen_port=turbogears.config.get('server.socket_port')),
+ '__requires__ = ["CherryPy < 3.0"]; import pkg_resources; '
+ 'from gunicorn.app.wsgiapp import run; run()',
+ '--bind', ':%s' % turbogears.config.get('server.socket_port'),
+ '--workers', '8', '--access-logfile', '-', '--preload',
+ 'bkr.server.wsgi:application'],
+ listen_port=turbogears.config.get('server.socket_port')),
])
processes.extend([
Process('slapd', args=['slapd', '-d0', '-F' + slapd_config_dir,
- '-hldap://127.0.0.1:3899/'],
+ '-hldap://127.0.0.1:3899/'],
listen_port=3899, stop_signal=signal.SIGINT),
])
try:
@@ -443,6 +477,7 @@ def setup_package():
process.stop()
raise
+
def teardown_package():
for process in processes:
process.stop()
diff --git a/Server/server.cfg b/Server/server.cfg
index b35a713..3c02e9c 100644
--- a/Server/server.cfg
+++ b/Server/server.cfg
@@ -137,7 +137,7 @@ sqlalchemy.pool_recycle = 3600
# Use OpenStack for running recipes on dynamically created guests.
# Beaker uses the credentials given here to authenticate on OpenStack,
# when creating OpenStack instances on behalf of users.
-#openstack.identity_api_url = 'https://openstack.example.com:13000/v3/'
+#openstack.identity_api_url = 'https://openstack.example.com:13000/v3'
#openstack.dashboard_url = 'https://openstack.example.com/dashboard/'
#openstack.username = ""
#openstack.password = ""
diff --git a/documentation/admin-guide/openstack.rst b/documentation/admin-guide/openstack.rst
index 4ad7928..379039f 100644
--- a/documentation/admin-guide/openstack.rst
+++ b/documentation/admin-guide/openstack.rst
@@ -43,7 +43,7 @@ dashboard (Horizon) URL and an OpenStack account of Beaker in :file:`/etc/beaker
# Use OpenStack for running recipes on dynamically created guests.
# Beaker uses the credentials given here to authenticate to OpenStack
# when creating OpenStack instances on behalf of users.
- openstack.identity_api_url = 'https://openstack.example.com:13000/v3.0'
+ openstack.identity_api_url = 'https://openstack.example.com:13000/v3'
openstack.dashboard_url = 'https://openstack.example.com/dashboard/'
openstack.username = ""
openstack.password = ""