buildbot / buildbot
Showing 4 of 65 files from the diff.
Other files ignored by Codecov
master/setup.py has changed.
.bbtravis.yml has changed.

@@ -25,9 +25,7 @@
Loading
25 25
26 26
27 27
import os
28 -
import re
29 28
30 -
import migrate
31 29
import sqlalchemy as sa
32 30
from sqlalchemy.engine import url
33 31
from sqlalchemy.pool import NullPool
@@ -134,22 +132,6 @@
Loading
134 132
                    super().should_retry(ex)])
135 133
136 134
137 -
def get_sqlalchemy_migrate_version():
138 -
    # sqlalchemy-migrate started including a version number in 0.7
139 -
    # Borrowed from model.py
140 -
    version = getattr(migrate, '__version__', 'old')
141 -
    if version == 'old':
142 -
        try:
143 -
            from migrate.versioning import schemadiff
144 -
            if hasattr(schemadiff, 'ColDiff'):
145 -
                version = "0.6.1"
146 -
            else:
147 -
                version = "0.6"
148 -
        except Exception:
149 -
            version = "0.0"
150 -
    return tuple(map(int, version.split('.')))
151 -
152 -
153 135
def sa_url_set_attr(u, attr, value):
154 136
    if hasattr(u, 'set'):
155 137
        return u.set(**{attr: value})
@@ -234,24 +216,6 @@
Loading
234 216
    return u, kwargs, None
235 217
236 218
237 -
def check_sqlalchemy_version():
238 -
    version = getattr(sa, '__version__', '0')
239 -
    try:
240 -
        version_digits = re.sub('[^0-9.]', '', version)
241 -
        version_tup = tuple(map(int, version_digits.split('.')))
242 -
    except TypeError:
243 -
        return  # unparseable -- oh well
244 -
245 -
    if version_tup < (0, 6):
246 -
        raise RuntimeError("SQLAlchemy version {} is too old".format(version))
247 -
    if version_tup > (0, 7, 10):
248 -
        mvt = get_sqlalchemy_migrate_version()
249 -
        if mvt < (0, 8, 0):
250 -
            raise RuntimeError(("SQLAlchemy version {} is not supported by "
251 -
                                "SQLAlchemy-Migrate version {}.{}.{}").format(version, mvt[0],
252 -
                                                                              mvt[1], mvt[2]))
253 -
254 -
255 219
def get_drivers_strategy(drivername):
256 220
    if drivername.startswith('sqlite'):
257 221
        return SqlLiteStrategy()
@@ -263,7 +227,6 @@
Loading
263 227
def create_engine(name_or_url, **kwargs):
264 228
    if 'basedir' not in kwargs:
265 229
        raise TypeError('no basedir supplied to create_engine')
266 -
    check_sqlalchemy_version()
267 230
268 231
    max_conns = None
269 232

@@ -14,10 +14,9 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
17 -
import migrate
18 -
import migrate.versioning.repository
17 +
import alembic
18 +
import alembic.config
19 19
import sqlalchemy as sa
20 -
from migrate import exceptions  # pylint: disable=ungrouped-imports
21 20
22 21
from twisted.internet import defer
23 22
from twisted.python import log
@@ -28,22 +27,27 @@
Loading
28 27
from buildbot.db.types.json import JsonObject
29 28
from buildbot.util import sautils
30 29
31 -
try:
32 -
    from migrate.versioning.schema import ControlledSchema  # pylint: disable=ungrouped-imports
33 -
except ImportError:
34 -
    ControlledSchema = None
35 30
36 -
37 -
class EightUpgradeError(Exception):
31 +
class UpgradeFromBefore0p9Error(Exception):
38 32
39 33
    def __init__(self):
40 -
        message = """You are trying to upgrade a buildbot 0.8.x master to buildbot 0.9.x
34 +
        message = """You are trying to upgrade a buildbot 0.8.x master to buildbot 0.9.x or newer.
41 35
        This is not supported. Please start from a clean database
42 -
        http://docs.buildbot.net/latest/manual/installation/nine-upgrade.html"""
36 +
        http://docs.buildbot.net/latest/manual/upgrading/0.9-upgrade.html"""
43 37
        # Call the base class constructor with the parameters it needs
44 38
        super().__init__(message)
45 39
46 40
41 +
class UpgradeFromBefore3p0Error(Exception):
42 +
43 +
    def __init__(self):
44 +
        message = """You are trying to upgrade to Buildbot 3.0 or newer from Buildbot 2.x or older.
45 +
        This is only supported via an intermediate upgrade to newest Buildbot 2.10.x that is
46 +
        available. Please first upgrade to 2.10.x and then try to upgrade to this version.
47 +
        http://docs.buildbot.net/latest/manual/upgrading/3.0-upgrade.html"""
48 +
        super().__init__(message)
49 +
50 +
47 51
class Model(base.DBConnectorComponent):
48 52
    #
49 53
    # schema
@@ -988,35 +992,51 @@
Loading
988 992
    # Migration support
989 993
    # -----------------
990 994
991 -
    # this is a bit more complicated than might be expected because the first
992 -
    # seven database versions were once implemented using a homespun migration
993 -
    # system, and we need to support upgrading masters from that system.  The
994 -
    # old system used a 'version' table, where SQLAlchemy-Migrate uses
995 -
    # 'migrate_version'
995 +
    # Buildbot has historically used 3 database migration systems:
996 +
    #  - homegrown system that used "version" table to track versions
997 +
    #  - SQLAlchemy-migrate that used "migrate_version" table to track versions
998 +
    #  - alembic that uses "alembic_version" table to track versions (current)
999 +
    # We need to detect each case and tell the user how to upgrade.
1000 +
1001 +
    config_path = util.sibpath(__file__, "migrations/alembic.ini")
996 1002
997 -
    repo_path = util.sibpath(__file__, "migrate")
1003 +
    def table_exists(self, conn, table):
1004 +
        try:
1005 +
            r = conn.execute(f"select * from {table} limit 1")
1006 +
            r.close()
1007 +
            return True
1008 +
        except Exception:
1009 +
            return False
1010 +
1011 +
    def migrate_get_version(self, conn):
1012 +
        r = conn.execute("select version from migrate_version limit 1")
1013 +
        version = r.scalar()
1014 +
        r.close()
1015 +
        return version
1016 +
1017 +
    def alembic_get_scripts(self):
1018 +
        alembic_config = alembic.config.Config(self.config_path)
1019 +
        return alembic.script.ScriptDirectory.from_config(alembic_config)
1020 +
1021 +
    def alembic_stamp(self, conn, alembic_scripts, revision):
1022 +
        context = alembic.runtime.migration.MigrationContext.configure(conn)
1023 +
        context.stamp(alembic_scripts, revision)
998 1024
999 1025
    @defer.inlineCallbacks
1000 1026
    def is_current(self):
1001 -
        if ControlledSchema is None:
1002 -
            # this should have been caught earlier by enginestrategy.py with a
1003 -
            # nicer error message
1004 -
            raise ImportError("SQLAlchemy/SQLAlchemy-Migrate version conflict")
1005 -
1006 -
        def thd(engine):
1007 -
            # we don't even have to look at the old version table - if there's
1008 -
            # no migrate_version, then we're not up to date.
1009 -
            repo = migrate.versioning.repository.Repository(self.repo_path)
1010 -
            repo_version = repo.latest
1011 -
            try:
1012 -
                # migrate.api doesn't let us hand in an engine
1013 -
                schema = ControlledSchema(engine, self.repo_path)
1014 -
                db_version = schema.version
1015 -
            except exceptions.DatabaseNotControlledError:
1027 +
        def thd(conn):
1028 +
            if not self.table_exists(conn, 'alembic_version'):
1016 1029
                return False
1017 1030
1018 -
            return db_version == repo_version
1019 -
        ret = yield self.db.pool.do_with_engine(thd)
1031 +
            alembic_scripts = self.alembic_get_scripts()
1032 +
            current_script_rev_head = alembic_scripts.get_current_head()
1033 +
1034 +
            context = alembic.runtime.migration.MigrationContext.configure(conn)
1035 +
            current_rev = context.get_current_revision()
1036 +
1037 +
            return current_rev == current_script_rev_head
1038 +
1039 +
        ret = yield self.db.pool.do(thd)
1020 1040
        return ret
1021 1041
1022 1042
    # returns a Deferred that returns None
@@ -1029,94 +1049,49 @@
Loading
1029 1049
    @defer.inlineCallbacks
1030 1050
    def upgrade(self):
1031 1051
1032 -
        # here, things are a little tricky.  If we have a 'version' table, then
1033 -
        # we need to version_control the database with the proper version
1034 -
        # number, drop 'version', and then upgrade.  If we have no 'version'
1035 -
        # table and no 'migrate_version' table, then we need to version_control
1036 -
        # the database.  Otherwise, we just need to upgrade it.
1037 -
1038 -
        def table_exists(engine, tbl):
1039 -
            try:
1040 -
                r = engine.execute("select * from {} limit 1".format(tbl))
1041 -
                r.close()
1042 -
                return True
1043 -
            except Exception:
1044 -
                return False
1052 +
        # the upgrade process must run in a db thread
1053 +
        def thd(conn):
1054 +
            alembic_scripts = self.alembic_get_scripts()
1055 +
            current_script_rev_head = alembic_scripts.get_current_head()
1045 1056
1046 -
        # http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100
1047 -
        # means  we cannot use the migrate.versioning.api module.  So these
1048 -
        # methods perform similar wrapping functions to what is done by the API
1049 -
        # functions, but without disposing of the engine.
1050 -
        def upgrade(engine):
1051 -
            schema = ControlledSchema(engine, self.repo_path)
1052 -
            changeset = schema.changeset(None)
1053 -
            with sautils.withoutSqliteForeignKeys(engine):
1054 -
                for version, change in changeset:
1055 -
                    log.msg('migrating schema version {} -> {}'.format(version, version + 1))
1056 -
                    schema.runchange(version, change, 1)
1057 -
1058 -
        def check_sqlalchemy_migrate_version():
1059 -
            # sqlalchemy-migrate started including a version number in 0.7; we
1060 -
            # support back to 0.6.1, but not 0.6.  We'll use some discovered
1061 -
            # differences between 0.6.1 and 0.6 to get that resolution.
1062 -
            version = getattr(migrate, '__version__', 'old')
1063 -
            if version == 'old':
1064 -
                try:
1065 -
                    from migrate.versioning import schemadiff
1066 -
                    if hasattr(schemadiff, 'ColDiff'):
1067 -
                        version = "0.6.1"
1068 -
                    else:
1069 -
                        version = "0.6"
1070 -
                except Exception:
1071 -
                    version = "0.0"
1072 -
            version_tup = tuple(map(int, version.split('-', 1)[0].split('.')))
1073 -
            log.msg("using SQLAlchemy-Migrate version {}".format(version))
1074 -
            if version_tup < (0, 6, 1):
1075 -
                raise RuntimeError(("You are using SQLAlchemy-Migrate {}. "
1076 -
                                    "The minimum version is 0.6.1.").format(version))
1077 -
1078 -
        def version_control(engine, version=None):
1079 -
            ControlledSchema.create(engine, self.repo_path, version)
1057 +
            if self.table_exists(conn, 'version'):
1058 +
                raise UpgradeFromBefore0p9Error()
1080 1059
1081 -
        # the upgrade process must run in a db thread
1082 -
        def thd(engine):
1083 -
            # if the migrate_version table exists, we can just let migrate
1084 -
            # take care of this process.
1085 -
            if table_exists(engine, 'migrate_version'):
1086 -
                r = engine.execute(
1087 -
                    "select version from migrate_version limit 1")
1088 -
                old_version = r.scalar()
1089 -
                if old_version < 40:
1090 -
                    raise EightUpgradeError()
1091 -
                try:
1092 -
                    upgrade(engine)
1093 -
                except sa.exc.NoSuchTableError as e:  # pragma: no cover
1094 -
                    if 'migration_tmp' in str(e):
1095 -
                        log.err('A serious error has been encountered during the upgrade. The '
1096 -
                                'previous upgrade has been likely interrupted. The database has '
1097 -
                                'been damaged and automatic recovery is impossible.')
1098 -
                        log.err('If you believe this is an error, please submit a bug to the '
1099 -
                                'Buildbot project.')
1100 -
                    raise
1101 -
1102 -
            # if the version table exists, then we can version_control things
1103 -
            # at that version, drop the version table, and let migrate take
1104 -
            # care of the rest.
1105 -
            elif table_exists(engine, 'version'):
1106 -
                raise EightUpgradeError()
1107 -
1108 -
            # otherwise, this db is new, so we don't bother using the migration engine
1109 -
            # and just create the tables, and put the version directly to
1110 -
            # latest
1111 -
            else:
1112 -
                # do some tests before getting started
1113 -
                test_unicode(engine)
1060 +
            if self.table_exists(conn, 'migrate_version'):
1061 +
                version = self.migrate_get_version(conn)
1062 +
1063 +
                if version < 40:
1064 +
                    raise UpgradeFromBefore0p9Error()
1065 +
1066 +
                last_sqlalchemy_migrate_version = 58
1067 +
                if version != last_sqlalchemy_migrate_version:
1068 +
                    raise UpgradeFromBefore3p0Error()
1114 1069
1070 +
                self.alembic_stamp(conn, alembic_scripts, alembic_scripts.get_base())
1071 +
                conn.execute('drop table migrate_version')
1072 +
1073 +
            if not self.table_exists(conn, 'alembic_version'):
1115 1074
                log.msg("Initializing empty database")
1116 -
                Model.metadata.create_all(engine)
1117 -
                repo = migrate.versioning.repository.Repository(self.repo_path)
1118 1075
1119 -
                version_control(engine, repo.latest)
1076 +
                # Do some tests first
1077 +
                test_unicode(conn)
1078 +
1079 +
                Model.metadata.create_all(conn)
1080 +
                self.alembic_stamp(conn, alembic_scripts, current_script_rev_head)
1081 +
                return
1082 +
1083 +
            context = alembic.runtime.migration.MigrationContext.configure(conn)
1084 +
            current_rev = context.get_current_revision()
1085 +
1086 +
            if current_rev == current_script_rev_head:
1087 +
                log.msg('Upgrading database: the current database schema is already the newest')
1088 +
                return
1089 +
1090 +
            log.msg('Upgrading database')
1091 +
            with sautils.withoutSqliteForeignKeys(conn):
1092 +
                with context.begin_transaction():
1093 +
                    context.run_migrations()
1094 +
1095 +
            log.msg('Upgrading database: done')
1120 1096
1121 -
        check_sqlalchemy_migrate_version()
1122 -
        yield self.db.pool.do_with_engine(thd)
1097 +
        yield self.db.pool.do(thd)

@@ -0,0 +1,23 @@
Loading
1 +
"""initial
2 +
3 +
Revision ID: 059
4 +
Revises: (none)
5 +
Create Date: 2021-09-07 20:00:00.000000
6 +
7 +
This empty Alembic revision is used as a placeholder revision for upgrades from older versions
8 +
of the database.
9 +
"""
10 +
11 +
# revision identifiers, used by Alembic.
12 +
revision = '059'
13 +
down_revision = None
14 +
branch_labels = None
15 +
depends_on = None
16 +
17 +
18 +
def upgrade():
19 +
    pass
20 +
21 +
22 +
def downgrade():
23 +
    pass
0 24
imilarity index 100%
1 25
ename from master/buildbot/db/migrate/versions/__init__.py
2 26
ename to master/buildbot/db/migrations/versions/__init__.py

@@ -132,7 +132,7 @@
Loading
132 132
                # Using in-memory database. Since it is reset after each process
133 133
                # restart, `buildbot upgrade-master` cannot be used (data is not
134 134
                # persistent). Upgrade model here to allow startup to continue.
135 -
                self.model.upgrade()
135 +
                yield self.model.upgrade()
136 136
            current = yield self.model.is_current()
137 137
            if not current:
138 138
                for l in upgrade_message.format(basedir=self.master.basedir).split('\n'):
Files Coverage
master/buildbot 92.28%
worker/buildbot_worker 85.52%
Project Totals (331 files) 91.88%
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading