install -d -m 0700 %{buildroot}%{_sysconfdir}/ipsilon
mv %{buildroot}/%{_bindir}/ipsilon %{buildroot}/%{_libexecdir}
mv %{buildroot}/%{_bindir}/ipsilon-server-install %{buildroot}/%{_sbindir}
+mv %{buildroot}/%{_bindir}/ipsilon-upgrade-database %{buildroot}/%{_sbindir}
mv %{buildroot}%{_defaultdocdir}/%{name} %{buildroot}%{_defaultdocdir}/%{name}-%{version}
rm -fr %{buildroot}%{python2_sitelib}/tests
ln -s %{_datadir}/fonts %{buildroot}%{_datadir}/ipsilon/ui/fonts
%files
%{_sbindir}/ipsilon-server-install
+%{_sbindir}/ipsilon-upgrade-database
%{_datadir}/ipsilon/templates/install/*.conf
%{_datadir}/ipsilon/ui/saml2sp
%dir %{python2_sitelib}/ipsilon/helpers
%{_defaultdocdir}/%{name}-%{version}
%{python2_sitelib}/ipsilon/admin
%{python2_sitelib}/ipsilon/rest
+%{python2_sitelib}/ipsilon/tools/dbupgrade.py*
%dir %{python2_sitelib}/ipsilon/login
%{python2_sitelib}/ipsilon/login/__init__*
%{python2_sitelib}/ipsilon/login/common*
+# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING
+
+import sys
+import os
+
+
+def find_config():
+ cfgfile = None
+ if (len(sys.argv) > 1):
+ cfgfile = sys.argv[-1]
+ elif os.path.isfile('ipsilon.conf'):
+ cfgfile = 'ipsilon.conf'
+ elif os.path.isfile('/etc/ipsilon/ipsilon.conf'):
+ cfgfile = '/etc/ipsilon/ipsilon.conf'
+ else:
+ raise IOError("Configuration file not found")
+ return cfgfile
from ipsilon.providers.common import ProvidersInstall
from ipsilon.helpers.common import EnvHelpersInstall
from ipsilon.util.data import UserStore
-from ipsilon.tools import files
+from ipsilon.tools import files, dbupgrade
import ConfigParser
import argparse
import cherrypy
'info_provider': {},
'auth_provider': {}}
- # Move pre-existing admin db away
+ # Move pre-existing dbs away
admin_db = cherrypy.config['admin.config.db']
if os.path.exists(admin_db):
shutil.move(admin_db, '%s.backup.%s' % (admin_db, now))
-
- # Rebuild user db
users_db = cherrypy.config['user.prefs.db']
if os.path.exists(users_db):
shutil.move(users_db, '%s.backup.%s' % (users_db, now))
+
+ # Initialize initial database schemas
+ dbupgrade.execute_upgrade(ipsilon_conf)
+
+ # Store primary admin
db = UserStore()
db.save_user_preferences(args['admin_user'], {'is_admin': 1})
with open(install_changes, 'w+') as f:
f.write(changes)
+ # Initialize extra database schemas
+ dbupgrade.execute_upgrade(ipsilon_conf)
+
# Fixup permissions so only the ipsilon user can read these files
files.fix_user_dirs(instance_conf, opts['system_user'])
files.fix_user_dirs(args['data_dir'], opts['system_user'])
--- /dev/null
+#!/usr/bin/python
+#
+# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING
+
+import sys
+from ipsilon import find_config
+from ipsilon.tools import dbupgrade
+
+
+try:
+ dbupgrade.execute_upgrade(find_config())
+except Exception as ex:
+ print ex
+ sys.exit(1)
import os
import atexit
import cherrypy
+from ipsilon import find_config
from ipsilon.util.data import AdminStore
from ipsilon.util import page
from ipsilon.root import Root
except Exception: # pylint: disable=broad-except
pass
-cfgfile = None
-if (len(sys.argv) > 1):
- cfgfile = sys.argv[-1]
-elif os.path.isfile('ipsilon.conf'):
- cfgfile = 'ipsilon.conf'
-elif os.path.isfile('/etc/ipsilon/ipsilon.conf'):
- cfgfile = '/etc/ipsilon/ipsilon.conf'
-else:
- raise IOError("Configuration file not found")
+cfgfile = find_config()
cherrypy.lib.sessions.SqlSession = ipsilon.util.sessions.SqlSession
cherrypy.config.update(cfgfile)
# Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING
-from ipsilon.util.data import Store
+from ipsilon.util.data import Store, UNIQUE_DATA_COLUMNS
from openid import oidutil
from openid.association import Association
if ((int(assocs[iden]['issued']) + int(assocs[iden]['lifetime']))
< time()):
self.del_unique_data('association', iden)
+
+ def _initialize_schema(self):
+ q = self._query(self._db, 'association', UNIQUE_DATA_COLUMNS,
+ trans=False)
+ q.create()
+
+ def _upgrade_schema(self, old_version):
+ raise NotImplementedError()
super(IdpProvider, self).__init__('openid', 'openid', *pargs)
self.mapping = InfoMapping()
self.page = None
+ self.datastore = None
self.server = None
self.basepath = None
self.extensions = LoadExtensions()
return self.page
+ def used_datastores(self):
+ return [self.datastore]
+
def init_idp(self):
+ self.datastore = OpenIDStore(self.get_config_value('database url'))
self.server = Server(
- OpenIDStore(self.get_config_value('database url')),
+ self.datastore,
op_endpoint=self.endpoint_url)
# Expose OpenID presence in the root
self.rest = Saml2RestBase(site, self)
return self.page
+ def used_datastores(self):
+ # pylint: disable=protected-access
+ return [self.sessionfactory._ss]
+
def init_idp(self):
idp = None
self.sessionfactory = SAMLSessionFactory(
--- /dev/null
+#!/usr/bin/python
+#
+# Copyright (C) 2015 Ipsilon project Contributors, for license see COPYING
+
+import cherrypy
+import os
+from jinja2 import Environment, FileSystemLoader
+import ipsilon.util.sessions
+from ipsilon.util.data import AdminStore, Store, UserStore, TranStore
+from ipsilon.util.sessions import SqlSession
+from ipsilon.root import Root
+
+
+def _upgrade_database(datastore):
+ print 'Considering datastore %s' % datastore.__class__.__name__
+ # pylint: disable=protected-access
+ current_version = datastore._get_schema_version()
+ # pylint: disable=protected-access
+ code_schema_version = datastore._code_schema_version()
+ upgrade_required = False
+ if current_version is None:
+ # Initialize schema
+ print 'Initializing schema for %s' % datastore.__class__.__name__
+ upgrade_required = True
+ elif current_version != code_schema_version:
+ print 'Upgrading schema for %s' % datastore.__class__.__name__
+ upgrade_required = True
+ else:
+ print 'Schema for %s is up-to-date' % datastore.__class__.__name__
+ if upgrade_required:
+ if datastore.is_readonly:
+ print 'Datastore is readonly. Please fix manually!'
+ return False
+ try:
+ datastore.upgrade_database()
+ except Exception as ex: # pylint: disable=broad-except
+ # Error upgrading database
+ print 'Error upgrading datastore: %s' % ex
+ return False
+ else:
+ # Upgrade went OK
+ return True
+ else:
+ return True
+
+
+def upgrade_failed():
+ print 'Upgrade failed. Please fix errors above and retry'
+ raise Exception('Upgrading failed')
+
+
+def execute_upgrade(cfgfile):
+ cherrypy.lib.sessions.SqlSession = ipsilon.util.sessions.SqlSession
+ cherrypy.config.update(cfgfile)
+
+ # pylint: disable=protected-access
+ Store._is_upgrade = True
+
+ datastore = AdminStore()
+ # First try to upgrade the config store before continuing
+ if not _upgrade_database(datastore):
+ return upgrade_failed()
+
+ admin_config = datastore.load_config()
+ for option in admin_config:
+ cherrypy.config[option] = admin_config[option]
+
+ # Initialize a minimal env
+ template_env = Environment(loader=FileSystemLoader(
+ os.path.join(cherrypy.config['base.dir'],
+ 'templates')))
+ root = Root('default', template_env)
+
+ # Handle the session store if that is Sql
+ print 'Handling sessions datastore'
+ if cherrypy.config['tools.sessions.storage_type'] != 'sql':
+ print 'Not SQL-based, skipping'
+ else:
+ dburi = cherrypy.config['tools.sessions.storage_dburi']
+ SqlSession.setup(storage_dburi=dburi)
+ if not _upgrade_database(SqlSession._store):
+ return upgrade_failed()
+
+ # Now handle the rest of the default datastores
+ for store in [UserStore, TranStore]:
+ store = store()
+ print 'Handling default datastore %s' % store.__class__.__name__
+ if not _upgrade_database(store):
+ return upgrade_failed()
+
+ # And now datastores for any of the plugins
+ for facility in ['provider_config',
+ 'login_config',
+ 'info_config']:
+ for plugin in root._site[facility].enabled:
+ print 'Handling plugin %s' % plugin
+ plugin = root._site[facility].available[plugin]
+ for store in plugin.used_datastores():
+ print 'Handling plugin datastore %s' % store.__class__.__name__
+ if not _upgrade_database(store):
+ return upgrade_failed()
+
+ # We are done with the init/upgrade
+ # pylint: disable=protected-access
+ Store._is_upgrade = False
UNIQUE_DATA_COLUMNS = ['uuid', 'name', 'value']
+class DatabaseError(Exception):
+ pass
+
+
class SqlStore(Log):
__instances = {}
return conn
-def SqlAutotable(f):
- def at(self, *args, **kwargs):
- self.create()
- return f(self, *args, **kwargs)
- return at
-
-
class SqlQuery(Log):
def __init__(self, db_obj, table, columns, trans=True):
def drop(self):
self._table.drop(checkfirst=True)
- @SqlAutotable
def select(self, kvfilter=None, columns=None):
return self._con.execute(select(self._columns(columns),
self._where(kvfilter)))
- @SqlAutotable
def insert(self, values):
self._con.execute(self._table.insert(values))
- @SqlAutotable
def update(self, values, kvfilter):
self._con.execute(self._table.update(self._where(kvfilter), values))
- @SqlAutotable
def delete(self, kvfilter):
self._con.execute(self._table.delete(self._where(kvfilter)))
class Store(Log):
+ _is_upgrade = False
+
def __init__(self, config_name=None, database_url=None):
if config_name is None and database_url is None:
raise ValueError('config_name or database_url must be provided')
else:
self._db = SqlStore.get_connection(name)
self._query = SqlQuery
- self._upgrade_database()
- def _upgrade_database(self):
+ if not self._is_upgrade:
+ self._check_database()
+
+ def _code_schema_version(self):
+ # This function makes it possible for separate plugins to have
+ # different schema versions. We default to the global schema
+ # version.
+ return CURRENT_SCHEMA_VERSION
+
+ def _get_schema_version(self):
+ # We are storing multiple versions: one per class
+ # That way, we can support plugins with differing schema versions from
+ # the main codebase, and even in the same database.
+ q = self._query(self._db, 'dbinfo', OPTIONS_COLUMNS, trans=False)
+ q.create()
+ cls_name = self.__class__.__name__
+ current_version = self.load_options('dbinfo').get('%s_schema'
+ % cls_name, {})
+ if 'version' in current_version:
+ return int(current_version['version'])
+ else:
+ # Also try the old table name.
+ # "scheme" was a typo, but we need to retain that now for compat
+ fallback_version = self.load_options('dbinfo').get('scheme',
+ {})
+ if 'version' in fallback_version:
+ return int(fallback_version['version'])
+ else:
+ return None
+
+ def _check_database(self):
if self.is_readonly:
# If the database is readonly, we cannot do anything to the
# schema. Let's just return, and assume people checked the
# upgrade notes
return
- current_version = self.load_options('dbinfo').get('scheme', None)
- if current_version is None or 'version' not in current_version:
- # No version stored, storing current version
- self.save_options('dbinfo', 'scheme',
- {'version': CURRENT_SCHEMA_VERSION})
- current_version = CURRENT_SCHEMA_VERSION
- else:
- current_version = int(current_version['version'])
- if current_version != CURRENT_SCHEMA_VERSION:
- self.debug('Upgrading database schema from %i to %i' % (
- current_version, CURRENT_SCHEMA_VERSION))
- self._upgrade_database_from(current_version)
-
- def _upgrade_database_from(self, old_schema_version):
- # Insert code here to upgrade from old_schema_version to
- # CURRENT_SCHEMA_VERSION
- raise Exception('Unable to upgrade database to current schema'
- ' version: version %i is unknown!' %
- old_schema_version)
+
+ current_version = self._get_schema_version()
+ if current_version is None:
+ self.error('Database initialization required! ' +
+ 'Please run ipsilon-upgrade-database')
+ raise DatabaseError('Database initialization required for %s' %
+ self.__class__.__name__)
+ if current_version != self._code_schema_version():
+ self.error('Database upgrade required! ' +
+ 'Please run ipsilon-upgrade-database')
+ raise DatabaseError('Database upgrade required for %s' %
+ self.__class__.__name__)
+
+ def _store_new_schema_version(self, new_version):
+ cls_name = self.__class__.__name__
+ self.save_options('dbinfo', '%s_schema' % cls_name,
+ {'version': new_version})
+
+ def _initialize_schema(self):
+ raise NotImplementedError()
+
+ def _upgrade_schema(self, old_version):
+ # Datastores need to figure out what to do with bigger old_versions
+ # themselves.
+ # They might implement downgrading if that's feasible, or just throw
+ # NotImplementedError
+ raise NotImplementedError()
+
+ def upgrade_database(self):
+ # Do whatever is needed to get schema to current version
+ old_schema_version = self._get_schema_version()
+ if old_schema_version is None:
+ # Just initialize a new schema
+ self._initialize_schema()
+ self._store_new_schema_version(self._code_schema_version())
+ elif old_schema_version != self._code_schema_version():
+ # Upgrade from old_schema_version to code_schema_version
+ self._upgrade_schema(old_schema_version)
+ self._store_new_schema_version(self._code_schema_version())
@property
def is_readonly(self):
table = plugin+"_data"
self._reset_data(table)
+ def _initialize_schema(self):
+ for table in ['config',
+ 'info_config',
+ 'login_config',
+ 'provider_config']:
+ q = self._query(self._db, table, OPTIONS_COLUMNS, trans=False)
+ q.create()
+
+ def _upgrade_schema(self, old_version):
+ raise NotImplementedError()
+
class UserStore(Store):
def load_plugin_data(self, plugin, user):
return self.load_options(plugin+"_data", user)
+ def _initialize_schema(self):
+ q = self._query(self._db, 'users', OPTIONS_COLUMNS, trans=False)
+ q.create()
+
+ def _upgrade_schema(self, old_version):
+ raise NotImplementedError()
+
class TranStore(Store):
def __init__(self, path=None):
super(TranStore, self).__init__('transactions.db')
+ def _initialize_schema(self):
+ q = self._query(self._db, 'transactions', UNIQUE_DATA_COLUMNS,
+ trans=False)
+ q.create()
+
+ def _upgrade_schema(self, old_version):
+ raise NotImplementedError()
+
class SAML2SessionStore(Store):
def wipe_data(self):
self._reset_data(self.table)
+
+ def _initialize_schema(self):
+ q = self._query(self._db, self.table, UNIQUE_DATA_COLUMNS,
+ trans=False)
+ q.create()
+
+ def _upgrade_schema(self, old_version):
+ raise NotImplementedError()
import cherrypy
import inspect
import logging
-from ipsilon.util.data import AdminStore
+from ipsilon.util.data import AdminStore, Store
from ipsilon.util.log import Log
return
self.refresh_plugin_config()
- self.on_enable()
+ is_upgrade = Store._is_upgrade # pylint: disable=protected-access
+ try:
+ Store._is_upgrade = True # pylint: disable=protected-access
+ self.on_enable()
+ for store in self.used_datastores():
+ store.upgrade_database()
+ finally:
+ Store._is_upgrade = is_upgrade # pylint: disable=protected-access
self.is_enabled = True
self.debug('Plugin enabled: %s' % self.name)
self.is_enabled = False
self.debug('Plugin disabled: %s' % self.name)
+ def used_datastores(self):
+ return []
+
def import_config(self, config):
self._config = config
import base64
from cherrypy.lib.sessions import Session
-from ipsilon.util.data import SqlStore, SqlQuery
+from ipsilon.util.data import Store, SqlQuery
import threading
try:
import cPickle as pickle
SESSION_COLUMNS = ['id', 'data', 'expiration_time']
+class SessionStore(Store):
+ def _initialize_schema(self):
+ q = self._query(self._db, 'sessions', SESSION_COLUMNS,
+ trans=False)
+ q.create()
+
+ def _upgrade_schema(self, old_version):
+ raise NotImplementedError()
+
+
class SqlSession(Session):
dburi = None
_db = None
+ _store = None
_proto = 2
locks = {}
if k == 'storage_dburi':
cls.dburi = v
- cls._db = SqlStore(cls.dburi)
+ cls._store = SessionStore(database_url=cls.dburi)
+ # pylint: disable=protected-access
+ cls._db = cls._store._db
def _exists(self):
q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS)
penv.update(os.environ)
penv['PYTHONPATH'] = os.getcwd()
+ schema_init = os.path.join(os.getcwd(), 'ipsilon/install/ipsilon-upgrade-database')
exe = os.path.join(os.getcwd(), 'ipsilon/ipsilon')
if args['cleanup']:
os.chdir(args['workdir'])
- p = subprocess.Popen([exe, conf], env=penv)
+ p = subprocess.Popen([schema_init, conf], env=penv)
p.wait()
+
+ if p.returncode == 0:
+ p = subprocess.Popen([exe, conf], env=penv)
+ p.wait()
glob('templates/admin/providers/*.html')),
],
scripts = ['ipsilon/ipsilon',
+ 'ipsilon/install/ipsilon-upgrade-database',
'ipsilon/install/ipsilon-server-install',
'ipsilon/install/ipsilon-client-install']
)
projects / cascardo / ipsilon.git / commitdiff