From ba59365931a4e35226b3d9be216d867ff1549846 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sun, 30 Aug 2015 12:55:21 +0200 Subject: [PATCH] Create database upgrade framework This creates a framework for uprading database scheme to the latest version, and a script that will execute them. Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- contrib/fedora/ipsilon.spec | 3 + ipsilon/__init__.py | 17 +++ ipsilon/install/ipsilon-server-install | 14 ++- ipsilon/install/ipsilon-upgrade-database | 14 +++ ipsilon/ipsilon | 11 +- ipsilon/providers/openid/store.py | 10 +- ipsilon/providers/openidp.py | 7 +- ipsilon/providers/saml2idp.py | 4 + ipsilon/tools/dbupgrade.py | 105 +++++++++++++++++ ipsilon/util/data.py | 142 ++++++++++++++++++----- ipsilon/util/plugin.py | 14 ++- ipsilon/util/sessions.py | 17 ++- quickrun.py | 7 +- setup.py | 1 + 14 files changed, 314 insertions(+), 52 deletions(-) create mode 100755 ipsilon/install/ipsilon-upgrade-database create mode 100644 ipsilon/tools/dbupgrade.py diff --git a/contrib/fedora/ipsilon.spec b/contrib/fedora/ipsilon.spec index 6553208..7da2ea0 100644 --- a/contrib/fedora/ipsilon.spec +++ b/contrib/fedora/ipsilon.spec @@ -239,6 +239,7 @@ install -d -m 0700 %{buildroot}%{_sharedstatedir}/ipsilon install -d -m 0700 %{buildroot}%{_sysconfdir}/ipsilon mv %{buildroot}/%{_bindir}/ipsilon %{buildroot}/%{_libexecdir} mv %{buildroot}/%{_bindir}/ipsilon-server-install %{buildroot}/%{_sbindir} +mv %{buildroot}/%{_bindir}/ipsilon-upgrade-database %{buildroot}/%{_sbindir} mv %{buildroot}%{_defaultdocdir}/%{name} %{buildroot}%{_defaultdocdir}/%{name}-%{version} rm -fr %{buildroot}%{python2_sitelib}/tests ln -s %{_datadir}/fonts %{buildroot}%{_datadir}/ipsilon/ui/fonts @@ -283,6 +284,7 @@ fi %files %{_sbindir}/ipsilon-server-install +%{_sbindir}/ipsilon-upgrade-database %{_datadir}/ipsilon/templates/install/*.conf %{_datadir}/ipsilon/ui/saml2sp %dir %{python2_sitelib}/ipsilon/helpers @@ -293,6 +295,7 @@ fi %{_defaultdocdir}/%{name}-%{version} %{python2_sitelib}/ipsilon/admin %{python2_sitelib}/ipsilon/rest +%{python2_sitelib}/ipsilon/tools/dbupgrade.py* %dir %{python2_sitelib}/ipsilon/login %{python2_sitelib}/ipsilon/login/__init__* %{python2_sitelib}/ipsilon/login/common* diff --git a/ipsilon/__init__.py b/ipsilon/__init__.py index e69de29..267e32b 100644 --- a/ipsilon/__init__.py +++ b/ipsilon/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING + +import sys +import os + + +def find_config(): + cfgfile = None + if (len(sys.argv) > 1): + cfgfile = sys.argv[-1] + elif os.path.isfile('ipsilon.conf'): + cfgfile = 'ipsilon.conf' + elif os.path.isfile('/etc/ipsilon/ipsilon.conf'): + cfgfile = '/etc/ipsilon/ipsilon.conf' + else: + raise IOError("Configuration file not found") + return cfgfile diff --git a/ipsilon/install/ipsilon-server-install b/ipsilon/install/ipsilon-server-install index 378479d..b4a9085 100755 --- a/ipsilon/install/ipsilon-server-install +++ b/ipsilon/install/ipsilon-server-install @@ -6,7 +6,7 @@ from ipsilon.info.common import InfoProviderInstall from ipsilon.providers.common import ProvidersInstall from ipsilon.helpers.common import EnvHelpersInstall from ipsilon.util.data import UserStore -from ipsilon.tools import files +from ipsilon.tools import files, dbupgrade import ConfigParser import argparse import cherrypy @@ -160,15 +160,18 @@ def install(plugins, args): 'info_provider': {}, 'auth_provider': {}} - # Move pre-existing admin db away + # Move pre-existing dbs away admin_db = cherrypy.config['admin.config.db'] if os.path.exists(admin_db): shutil.move(admin_db, '%s.backup.%s' % (admin_db, now)) - - # Rebuild user db users_db = cherrypy.config['user.prefs.db'] if os.path.exists(users_db): shutil.move(users_db, '%s.backup.%s' % (users_db, now)) + + # Initialize initial database schemas + dbupgrade.execute_upgrade(ipsilon_conf) + + # Store primary admin db = UserStore() db.save_user_preferences(args['admin_user'], {'is_admin': 1}) @@ -217,6 +220,9 @@ def install(plugins, args): with open(install_changes, 'w+') as f: f.write(changes) + # Initialize extra database schemas + dbupgrade.execute_upgrade(ipsilon_conf) + # Fixup permissions so only the ipsilon user can read these files files.fix_user_dirs(instance_conf, opts['system_user']) files.fix_user_dirs(args['data_dir'], opts['system_user']) diff --git a/ipsilon/install/ipsilon-upgrade-database b/ipsilon/install/ipsilon-upgrade-database new file mode 100755 index 0000000..7ee18fb --- /dev/null +++ b/ipsilon/install/ipsilon-upgrade-database @@ -0,0 +1,14 @@ +#!/usr/bin/python +# +# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING + +import sys +from ipsilon import find_config +from ipsilon.tools import dbupgrade + + +try: + dbupgrade.execute_upgrade(find_config()) +except Exception as ex: + print ex + sys.exit(1) diff --git a/ipsilon/ipsilon b/ipsilon/ipsilon index bec13af..8264bce 100755 --- a/ipsilon/ipsilon +++ b/ipsilon/ipsilon @@ -8,6 +8,7 @@ import glob import os import atexit import cherrypy +from ipsilon import find_config from ipsilon.util.data import AdminStore from ipsilon.util import page from ipsilon.root import Root @@ -27,15 +28,7 @@ def nuke_session_locks(): except Exception: # pylint: disable=broad-except pass -cfgfile = None -if (len(sys.argv) > 1): - cfgfile = sys.argv[-1] -elif os.path.isfile('ipsilon.conf'): - cfgfile = 'ipsilon.conf' -elif os.path.isfile('/etc/ipsilon/ipsilon.conf'): - cfgfile = '/etc/ipsilon/ipsilon.conf' -else: - raise IOError("Configuration file not found") +cfgfile = find_config() cherrypy.lib.sessions.SqlSession = ipsilon.util.sessions.SqlSession cherrypy.config.update(cfgfile) diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py index 6443771..e759bca 100644 --- a/ipsilon/providers/openid/store.py +++ b/ipsilon/providers/openid/store.py @@ -1,6 +1,6 @@ # Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING -from ipsilon.util.data import Store +from ipsilon.util.data import Store, UNIQUE_DATA_COLUMNS from openid import oidutil from openid.association import Association @@ -77,3 +77,11 @@ class OpenIDStore(Store, OpenIDStoreInterface): if ((int(assocs[iden]['issued']) + int(assocs[iden]['lifetime'])) < time()): self.del_unique_data('association', iden) + + def _initialize_schema(self): + q = self._query(self._db, 'association', UNIQUE_DATA_COLUMNS, + trans=False) + q.create() + + def _upgrade_schema(self, old_version): + raise NotImplementedError() diff --git a/ipsilon/providers/openidp.py b/ipsilon/providers/openidp.py index dc47faf..21e48a3 100644 --- a/ipsilon/providers/openidp.py +++ b/ipsilon/providers/openidp.py @@ -19,6 +19,7 @@ class IdpProvider(ProviderBase): super(IdpProvider, self).__init__('openid', 'openid', *pargs) self.mapping = InfoMapping() self.page = None + self.datastore = None self.server = None self.basepath = None self.extensions = LoadExtensions() @@ -109,9 +110,13 @@ Provides OpenID 2.0 authentication infrastructure. """ return self.page + def used_datastores(self): + return [self.datastore] + def init_idp(self): + self.datastore = OpenIDStore(self.get_config_value('database url')) self.server = Server( - OpenIDStore(self.get_config_value('database url')), + self.datastore, op_endpoint=self.endpoint_url) # Expose OpenID presence in the root diff --git a/ipsilon/providers/saml2idp.py b/ipsilon/providers/saml2idp.py index 590b638..4c2639f 100644 --- a/ipsilon/providers/saml2idp.py +++ b/ipsilon/providers/saml2idp.py @@ -342,6 +342,10 @@ Provides SAML 2.0 authentication infrastructure. """ self.rest = Saml2RestBase(site, self) return self.page + def used_datastores(self): + # pylint: disable=protected-access + return [self.sessionfactory._ss] + def init_idp(self): idp = None self.sessionfactory = SAMLSessionFactory( diff --git a/ipsilon/tools/dbupgrade.py b/ipsilon/tools/dbupgrade.py new file mode 100644 index 0000000..16eebb6 --- /dev/null +++ b/ipsilon/tools/dbupgrade.py @@ -0,0 +1,105 @@ +#!/usr/bin/python +# +# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING + +import cherrypy +import os +from jinja2 import Environment, FileSystemLoader +import ipsilon.util.sessions +from ipsilon.util.data import AdminStore, Store, UserStore, TranStore +from ipsilon.util.sessions import SqlSession +from ipsilon.root import Root + + +def _upgrade_database(datastore): + print 'Considering datastore %s' % datastore.__class__.__name__ + # pylint: disable=protected-access + current_version = datastore._get_schema_version() + # pylint: disable=protected-access + code_schema_version = datastore._code_schema_version() + upgrade_required = False + if current_version is None: + # Initialize schema + print 'Initializing schema for %s' % datastore.__class__.__name__ + upgrade_required = True + elif current_version != code_schema_version: + print 'Upgrading schema for %s' % datastore.__class__.__name__ + upgrade_required = True + else: + print 'Schema for %s is up-to-date' % datastore.__class__.__name__ + if upgrade_required: + if datastore.is_readonly: + print 'Datastore is readonly. Please fix manually!' + return False + try: + datastore.upgrade_database() + except Exception as ex: # pylint: disable=broad-except + # Error upgrading database + print 'Error upgrading datastore: %s' % ex + return False + else: + # Upgrade went OK + return True + else: + return True + + +def upgrade_failed(): + print 'Upgrade failed. Please fix errors above and retry' + raise Exception('Upgrading failed') + + +def execute_upgrade(cfgfile): + cherrypy.lib.sessions.SqlSession = ipsilon.util.sessions.SqlSession + cherrypy.config.update(cfgfile) + + # pylint: disable=protected-access + Store._is_upgrade = True + + datastore = AdminStore() + # First try to upgrade the config store before continuing + if not _upgrade_database(datastore): + return upgrade_failed() + + admin_config = datastore.load_config() + for option in admin_config: + cherrypy.config[option] = admin_config[option] + + # Initialize a minimal env + template_env = Environment(loader=FileSystemLoader( + os.path.join(cherrypy.config['base.dir'], + 'templates'))) + root = Root('default', template_env) + + # Handle the session store if that is Sql + print 'Handling sessions datastore' + if cherrypy.config['tools.sessions.storage_type'] != 'sql': + print 'Not SQL-based, skipping' + else: + dburi = cherrypy.config['tools.sessions.storage_dburi'] + SqlSession.setup(storage_dburi=dburi) + if not _upgrade_database(SqlSession._store): + return upgrade_failed() + + # Now handle the rest of the default datastores + for store in [UserStore, TranStore]: + store = store() + print 'Handling default datastore %s' % store.__class__.__name__ + if not _upgrade_database(store): + return upgrade_failed() + + # And now datastores for any of the plugins + for facility in ['provider_config', + 'login_config', + 'info_config']: + for plugin in root._site[facility].enabled: + print 'Handling plugin %s' % plugin + plugin = root._site[facility].available[plugin] + for store in plugin.used_datastores(): + print 'Handling plugin datastore %s' % store.__class__.__name__ + if not _upgrade_database(store): + return upgrade_failed() + + # We are done with the init/upgrade + # pylint: disable=protected-access + Store._is_upgrade = False diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index 3c116bb..200feb8 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -18,6 +18,10 @@ OPTIONS_COLUMNS = ['name', 'option', 'value'] UNIQUE_DATA_COLUMNS = ['uuid', 'name', 'value'] +class DatabaseError(Exception): + pass + + class SqlStore(Log): __instances = {} @@ -68,13 +72,6 @@ class SqlStore(Log): return conn -def SqlAutotable(f): - def at(self, *args, **kwargs): - self.create() - return f(self, *args, **kwargs) - return at - - class SqlQuery(Log): def __init__(self, db_obj, table, columns, trans=True): @@ -122,20 +119,16 @@ class SqlQuery(Log): def drop(self): self._table.drop(checkfirst=True) - @SqlAutotable def select(self, kvfilter=None, columns=None): return self._con.execute(select(self._columns(columns), self._where(kvfilter))) - @SqlAutotable def insert(self, values): self._con.execute(self._table.insert(values)) - @SqlAutotable def update(self, values, kvfilter): self._con.execute(self._table.update(self._where(kvfilter), values)) - @SqlAutotable def delete(self, kvfilter): self._con.execute(self._table.delete(self._where(kvfilter))) @@ -253,6 +246,8 @@ class FileQuery(Log): class Store(Log): + _is_upgrade = False + def __init__(self, config_name=None, database_url=None): if config_name is None and database_url is None: raise ValueError('config_name or database_url must be provided') @@ -269,33 +264,82 @@ class Store(Log): else: self._db = SqlStore.get_connection(name) self._query = SqlQuery - self._upgrade_database() - def _upgrade_database(self): + if not self._is_upgrade: + self._check_database() + + def _code_schema_version(self): + # This function makes it possible for separate plugins to have + # different schema versions. We default to the global schema + # version. + return CURRENT_SCHEMA_VERSION + + def _get_schema_version(self): + # We are storing multiple versions: one per class + # That way, we can support plugins with differing schema versions from + # the main codebase, and even in the same database. + q = self._query(self._db, 'dbinfo', OPTIONS_COLUMNS, trans=False) + q.create() + cls_name = self.__class__.__name__ + current_version = self.load_options('dbinfo').get('%s_schema' + % cls_name, {}) + if 'version' in current_version: + return int(current_version['version']) + else: + # Also try the old table name. + # "scheme" was a typo, but we need to retain that now for compat + fallback_version = self.load_options('dbinfo').get('scheme', + {}) + if 'version' in fallback_version: + return int(fallback_version['version']) + else: + return None + + def _check_database(self): if self.is_readonly: # If the database is readonly, we cannot do anything to the # schema. Let's just return, and assume people checked the # upgrade notes return - current_version = self.load_options('dbinfo').get('scheme', None) - if current_version is None or 'version' not in current_version: - # No version stored, storing current version - self.save_options('dbinfo', 'scheme', - {'version': CURRENT_SCHEMA_VERSION}) - current_version = CURRENT_SCHEMA_VERSION - else: - current_version = int(current_version['version']) - if current_version != CURRENT_SCHEMA_VERSION: - self.debug('Upgrading database schema from %i to %i' % ( - current_version, CURRENT_SCHEMA_VERSION)) - self._upgrade_database_from(current_version) - - def _upgrade_database_from(self, old_schema_version): - # Insert code here to upgrade from old_schema_version to - # CURRENT_SCHEMA_VERSION - raise Exception('Unable to upgrade database to current schema' - ' version: version %i is unknown!' % - old_schema_version) + + current_version = self._get_schema_version() + if current_version is None: + self.error('Database initialization required! ' + + 'Please run ipsilon-upgrade-database') + raise DatabaseError('Database initialization required for %s' % + self.__class__.__name__) + if current_version != self._code_schema_version(): + self.error('Database upgrade required! ' + + 'Please run ipsilon-upgrade-database') + raise DatabaseError('Database upgrade required for %s' % + self.__class__.__name__) + + def _store_new_schema_version(self, new_version): + cls_name = self.__class__.__name__ + self.save_options('dbinfo', '%s_schema' % cls_name, + {'version': new_version}) + + def _initialize_schema(self): + raise NotImplementedError() + + def _upgrade_schema(self, old_version): + # Datastores need to figure out what to do with bigger old_versions + # themselves. + # They might implement downgrading if that's feasible, or just throw + # NotImplementedError + raise NotImplementedError() + + def upgrade_database(self): + # Do whatever is needed to get schema to current version + old_schema_version = self._get_schema_version() + if old_schema_version is None: + # Just initialize a new schema + self._initialize_schema() + self._store_new_schema_version(self._code_schema_version()) + elif old_schema_version != self._code_schema_version(): + # Upgrade from old_schema_version to code_schema_version + self._upgrade_schema(old_schema_version) + self._store_new_schema_version(self._code_schema_version()) @property def is_readonly(self): @@ -487,6 +531,17 @@ class AdminStore(Store): table = plugin+"_data" self._reset_data(table) + def _initialize_schema(self): + for table in ['config', + 'info_config', + 'login_config', + 'provider_config']: + q = self._query(self._db, table, OPTIONS_COLUMNS, trans=False) + q.create() + + def _upgrade_schema(self, old_version): + raise NotImplementedError() + class UserStore(Store): @@ -505,12 +560,27 @@ class UserStore(Store): def load_plugin_data(self, plugin, user): return self.load_options(plugin+"_data", user) + def _initialize_schema(self): + q = self._query(self._db, 'users', OPTIONS_COLUMNS, trans=False) + q.create() + + def _upgrade_schema(self, old_version): + raise NotImplementedError() + class TranStore(Store): def __init__(self, path=None): super(TranStore, self).__init__('transactions.db') + def _initialize_schema(self): + q = self._query(self._db, 'transactions', UNIQUE_DATA_COLUMNS, + trans=False) + q.create() + + def _upgrade_schema(self, old_version): + raise NotImplementedError() + class SAML2SessionStore(Store): @@ -593,3 +663,11 @@ class SAML2SessionStore(Store): def wipe_data(self): self._reset_data(self.table) + + def _initialize_schema(self): + q = self._query(self._db, self.table, UNIQUE_DATA_COLUMNS, + trans=False) + q.create() + + def _upgrade_schema(self, old_version): + raise NotImplementedError() diff --git a/ipsilon/util/plugin.py b/ipsilon/util/plugin.py index 1f66d8e..978e470 100644 --- a/ipsilon/util/plugin.py +++ b/ipsilon/util/plugin.py @@ -5,7 +5,7 @@ import imp import cherrypy import inspect import logging -from ipsilon.util.data import AdminStore +from ipsilon.util.data import AdminStore, Store from ipsilon.util.log import Log @@ -146,7 +146,14 @@ class PluginObject(Log): return self.refresh_plugin_config() - self.on_enable() + is_upgrade = Store._is_upgrade # pylint: disable=protected-access + try: + Store._is_upgrade = True # pylint: disable=protected-access + self.on_enable() + for store in self.used_datastores(): + store.upgrade_database() + finally: + Store._is_upgrade = is_upgrade # pylint: disable=protected-access self.is_enabled = True self.debug('Plugin enabled: %s' % self.name) @@ -159,6 +166,9 @@ class PluginObject(Log): self.is_enabled = False self.debug('Plugin disabled: %s' % self.name) + def used_datastores(self): + return [] + def import_config(self, config): self._config = config diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py index f5390dc..b870319 100644 --- a/ipsilon/util/sessions.py +++ b/ipsilon/util/sessions.py @@ -2,7 +2,7 @@ import base64 from cherrypy.lib.sessions import Session -from ipsilon.util.data import SqlStore, SqlQuery +from ipsilon.util.data import Store, SqlQuery import threading try: import cPickle as pickle @@ -13,10 +13,21 @@ except ImportError: SESSION_COLUMNS = ['id', 'data', 'expiration_time'] +class SessionStore(Store): + def _initialize_schema(self): + q = self._query(self._db, 'sessions', SESSION_COLUMNS, + trans=False) + q.create() + + def _upgrade_schema(self, old_version): + raise NotImplementedError() + + class SqlSession(Session): dburi = None _db = None + _store = None _proto = 2 locks = {} @@ -28,7 +39,9 @@ class SqlSession(Session): if k == 'storage_dburi': cls.dburi = v - cls._db = SqlStore(cls.dburi) + cls._store = SessionStore(database_url=cls.dburi) + # pylint: disable=protected-access + cls._db = cls._store._db def _exists(self): q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS) diff --git a/quickrun.py b/quickrun.py index c565fcb..73baae7 100755 --- a/quickrun.py +++ b/quickrun.py @@ -80,6 +80,7 @@ if __name__ == '__main__': penv.update(os.environ) penv['PYTHONPATH'] = os.getcwd() + schema_init = os.path.join(os.getcwd(), 'ipsilon/install/ipsilon-upgrade-database') exe = os.path.join(os.getcwd(), 'ipsilon/ipsilon') if args['cleanup']: @@ -97,5 +98,9 @@ if __name__ == '__main__': os.chdir(args['workdir']) - p = subprocess.Popen([exe, conf], env=penv) + p = subprocess.Popen([schema_init, conf], env=penv) p.wait() + + if p.returncode == 0: + p = subprocess.Popen([exe, conf], env=penv) + p.wait() diff --git a/setup.py b/setup.py index 7e69d0f..eec94e9 100755 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup( glob('templates/admin/providers/*.html')), ], scripts = ['ipsilon/ipsilon', + 'ipsilon/install/ipsilon-upgrade-database', 'ipsilon/install/ipsilon-server-install', 'ipsilon/install/ipsilon-client-install'] ) -- 2.20.1