Add database migration code

CHANGES.md

@@ -14,6 +14,7 @@
 * Fix multiple instances of SG being able to start
 * Fix garbled text appearing during startup in console
 * Fix startup code order and general re-factoring (adapted from midgetspy/Sick-Beard)
+* Add database migration code
 
 [develop changelog]
 * Add TVRage network name standardization

sickbeard/__init__.py

@@ -1069,7 +1069,7 @@ def initialize(consoleLogging=True):
 
         # initialize the main SB database
         myDB = db.DBConnection()
-        db.upgradeDatabase(myDB, mainDB.InitialSchema)
+        db.MigrationCode(myDB)
 
         # initialize the cache database
         myDB = db.DBConnection('cache.db')

sickbeard/databases/mainDB.py

@@ -27,7 +27,8 @@ from sickbeard import encodingKludge as ek
 from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
 
 MIN_DB_VERSION = 9  # oldest db version we support migrating from
-MAX_DB_VERSION = 40
+MAX_DB_VERSION = 20000
+
 
 class MainSanityCheck(db.DBSanityCheck):
     def check(self):
@@ -155,11 +156,12 @@ def backupDatabase(version):
 # ======================
 # Add new migrations at the bottom of the list; subclass the previous migration.
 
-class InitialSchema(db.SchemaUpgrade):
-    def test(self):
-        return self.hasTable("db_version")
 
+# 0 -> 31
+class InitialSchema(db.SchemaUpgrade):
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         if not self.hasTable("tv_shows") and not self.hasTable("db_version"):
             queries = [
                 "CREATE TABLE db_version (db_version INTEGER);",
@@ -197,14 +199,14 @@ class InitialSchema(db.SchemaUpgrade):
                                           "If you have used other forks of SickGear, your database may be unusable due to their modifications."
                 )
 
+        return self.checkDBVersion()
 
-class AddSizeAndSceneNameFields(InitialSchema):
-    def test(self):
-        return self.checkDBVersion() >= 10
 
+# 9 -> 10
+class AddSizeAndSceneNameFields(db.SchemaUpgrade):
     def execute(self):
 
-        backupDatabase(10)
+        backupDatabase(self.checkDBVersion())
 
         if not self.hasColumn("tv_episodes", "file_size"):
             self.addColumn("tv_episodes", "file_size")
@@ -308,13 +310,14 @@ class AddSizeAndSceneNameFields(InitialSchema):
                                    [ep_file_name, cur_result["episode_id"]])
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class RenameSeasonFolders(AddSizeAndSceneNameFields):
+# 10 -> 11
-    def test(self):
+class RenameSeasonFolders(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 11
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         # rename the column
         self.connection.action("ALTER TABLE tv_shows RENAME TO tmp_tv_shows")
         self.connection.action(
@@ -329,9 +332,11 @@ class RenameSeasonFolders(AddSizeAndSceneNameFields):
         self.connection.action("DROP TABLE tmp_tv_shows")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class Add1080pAndRawHDQualities(RenameSeasonFolders):
+# 11 -> 12
+class Add1080pAndRawHDQualities(db.SchemaUpgrade):
     """Add support for 1080p related qualities along with RawHD
 
     Quick overview of what the upgrade needs to do:
@@ -347,9 +352,6 @@ class Add1080pAndRawHDQualities(RenameSeasonFolders):
         fullhdwebdl  |      | 1<<6
     """
 
-    def test(self):
-        return self.checkDBVersion() >= 12
-
     def _update_status(self, old_status):
         (status, quality) = common.Quality.splitCompositeStatus(old_status)
         return common.Quality.compositeStatus(status, self._update_quality(quality))
@@ -464,16 +466,15 @@ class Add1080pAndRawHDQualities(RenameSeasonFolders):
         # cleanup and reduce db if any previous data was removed
         logger.log(u"Performing a vacuum on the database.", logger.DEBUG)
         self.connection.action("VACUUM")
+        return self.checkDBVersion()
 
 
-class AddShowidTvdbidIndex(Add1080pAndRawHDQualities):
+# 12 -> 13
+class AddShowidTvdbidIndex(db.SchemaUpgrade):
     """ Adding index on tvdb_id (tv_shows) and showid (tv_episodes) to speed up searches/queries """
 
-    def test(self):
-        return self.checkDBVersion() >= 13
-
     def execute(self):
-        backupDatabase(13)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Check for duplicate shows before adding unique index.")
         MainSanityCheck(self.connection).fix_duplicate_shows('tvdb_id')
@@ -485,16 +486,14 @@ class AddShowidTvdbidIndex(Add1080pAndRawHDQualities):
             self.connection.action("CREATE UNIQUE INDEX idx_tvdb_id ON tv_shows (tvdb_id);")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddLastUpdateTVDB(AddShowidTvdbidIndex):
+# 13 -> 14
+class AddLastUpdateTVDB(db.SchemaUpgrade):
     """ Adding column last_update_tvdb to tv_shows for controlling nightly updates """
-
-    def test(self):
-        return self.checkDBVersion() >= 14
-
     def execute(self):
-        backupDatabase(14)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column last_update_tvdb to tvshows")
         if not self.hasColumn("tv_shows", "last_update_tvdb"):
@@ -503,19 +502,21 @@ class AddLastUpdateTVDB(AddShowidTvdbidIndex):
         self.incDBVersion()
 
 
-class AddDBIncreaseTo15(AddLastUpdateTVDB):
+# 14 -> 15
-    def test(self):
+class AddDBIncreaseTo15(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 15
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddIMDbInfo(AddDBIncreaseTo15):
+# 15 -> 16
-    def test(self):
+class AddIMDbInfo(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 16
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
+
         self.connection.action(
             "CREATE TABLE imdb_info (tvdb_id INTEGER PRIMARY KEY, imdb_id TEXT, title TEXT, year NUMERIC, akas TEXT, runtimes NUMERIC, genres TEXT, countries TEXT, country_codes TEXT, certificates TEXT, rating TEXT, votes INTEGER, last_update NUMERIC)")
 
@@ -523,71 +524,73 @@ class AddIMDbInfo(AddDBIncreaseTo15):
             self.addColumn("tv_shows", "imdb_id")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddProperNamingSupport(AddIMDbInfo):
+# 16 -> 17
-    def test(self):
+class AddProperNamingSupport(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 17
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         self.addColumn("tv_episodes", "is_proper")
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddEmailSubscriptionTable(AddProperNamingSupport):
+# 17 -> 18
-    def test(self):
+class AddEmailSubscriptionTable(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 18
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         self.addColumn('tv_shows', 'notify_list', 'TEXT', None)
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddProperSearch(AddEmailSubscriptionTable):
+# 18 -> 19
-    def test(self):
+class AddProperSearch(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 19
-
     def execute(self):
-        backupDatabase(19)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column last_proper_search to info")
         if not self.hasColumn("info", "last_proper_search"):
             self.addColumn("info", "last_proper_search", default=1)
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddDvdOrderOption(AddProperSearch):
+# 19 -> 20
-    def test(self):
+class AddDvdOrderOption(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 20
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         logger.log(u"Adding column dvdorder to tvshows")
         if not self.hasColumn("tv_shows", "dvdorder"):
             self.addColumn("tv_shows", "dvdorder", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddSubtitlesSupport(AddDvdOrderOption):
+# 20 -> 21
-    def test(self):
+class AddSubtitlesSupport(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 21
-
     def execute(self):
+        backupDatabase(self.checkDBVersion())
+
         if not self.hasColumn("tv_shows", "subtitles"):
             self.addColumn("tv_shows", "subtitles")
             self.addColumn("tv_episodes", "subtitles", "TEXT", "")
             self.addColumn("tv_episodes", "subtitles_searchcount")
             self.addColumn("tv_episodes", "subtitles_lastsearch", "TIMESTAMP", str(datetime.datetime.min))
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class ConvertTVShowsToIndexerScheme(AddSubtitlesSupport):
+# 21 -> 22
-    def test(self):
+class ConvertTVShowsToIndexerScheme(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 22
-
     def execute(self):
-        backupDatabase(22)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Converting TV Shows table to Indexer Scheme...")
 
@@ -608,14 +611,13 @@ class ConvertTVShowsToIndexerScheme(AddSubtitlesSupport):
         self.connection.action("UPDATE tv_shows SET indexer = 1")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class ConvertTVEpisodesToIndexerScheme(ConvertTVShowsToIndexerScheme):
+# 22 -> 23
-    def test(self):
+class ConvertTVEpisodesToIndexerScheme(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 23
-
     def execute(self):
-        backupDatabase(23)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Converting TV Episodes table to Indexer Scheme...")
 
@@ -639,14 +641,13 @@ class ConvertTVEpisodesToIndexerScheme(ConvertTVShowsToIndexerScheme):
         self.connection.action("UPDATE tv_episodes SET indexer = 1")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class ConvertIMDBInfoToIndexerScheme(ConvertTVEpisodesToIndexerScheme):
+# 23 -> 24
-    def test(self):
+class ConvertIMDBInfoToIndexerScheme(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 24
-
     def execute(self):
-        backupDatabase(24)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Converting IMDB Info table to Indexer Scheme...")
 
@@ -662,14 +663,13 @@ class ConvertIMDBInfoToIndexerScheme(ConvertTVEpisodesToIndexerScheme):
         self.connection.action("DROP TABLE tmp_imdb_info")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class ConvertInfoToIndexerScheme(ConvertIMDBInfoToIndexerScheme):
+# 24 -> 25
-    def test(self):
+class ConvertInfoToIndexerScheme(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 25
-
     def execute(self):
-        backupDatabase(25)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Converting Info table to Indexer Scheme...")
 
@@ -685,28 +685,27 @@ class ConvertInfoToIndexerScheme(ConvertIMDBInfoToIndexerScheme):
         self.connection.action("DROP TABLE tmp_info")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddArchiveFirstMatchOption(ConvertInfoToIndexerScheme):
+# 25 -> 26
-    def test(self):
+class AddArchiveFirstMatchOption(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 26
-
     def execute(self):
-        backupDatabase(26)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column archive_firstmatch to tvshows")
         if not self.hasColumn("tv_shows", "archive_firstmatch"):
             self.addColumn("tv_shows", "archive_firstmatch", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddSceneNumbering(AddArchiveFirstMatchOption):
+# 26 -> 27
-    def test(self):
+class AddSceneNumbering(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 27
 
     def execute(self):
-        backupDatabase(27)
+        backupDatabase(self.checkDBVersion())
 
         if self.hasTable("scene_numbering"):
             self.connection.action("DROP TABLE scene_numbering")
@@ -715,14 +714,13 @@ class AddSceneNumbering(AddArchiveFirstMatchOption):
             "CREATE TABLE scene_numbering (indexer TEXT, indexer_id INTEGER, season INTEGER, episode INTEGER, scene_season INTEGER, scene_episode INTEGER, PRIMARY KEY (indexer_id, season, episode, scene_season, scene_episode))")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class ConvertIndexerToInteger(AddSceneNumbering):
+# 27 -> 28
-    def test(self):
+class ConvertIndexerToInteger(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 28
-
     def execute(self):
-        backupDatabase(28)
+        backupDatabase(self.checkDBVersion())
 
         cl = []
         logger.log(u"Converting Indexer to Integer ...", logger.MESSAGE)
@@ -736,16 +734,15 @@ class ConvertIndexerToInteger(AddSceneNumbering):
         self.connection.mass_action(cl)
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddRequireAndIgnoreWords(ConvertIndexerToInteger):
+# 28 -> 29
+class AddRequireAndIgnoreWords(db.SchemaUpgrade):
     """ Adding column rls_require_words and rls_ignore_words to tv_shows """
 
-    def test(self):
-        return self.checkDBVersion() >= 29
-
     def execute(self):
-        backupDatabase(29)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column rls_require_words to tvshows")
         if not self.hasColumn("tv_shows", "rls_require_words"):
@@ -756,14 +753,13 @@ class AddRequireAndIgnoreWords(ConvertIndexerToInteger):
             self.addColumn("tv_shows", "rls_ignore_words", "TEXT", "")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddSportsOption(AddRequireAndIgnoreWords):
+# 29 -> 30
-    def test(self):
+class AddSportsOption(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 30
-
     def execute(self):
-        backupDatabase(30)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column sports to tvshows")
         if not self.hasColumn("tv_shows", "sports"):
@@ -782,65 +778,63 @@ class AddSportsOption(AddRequireAndIgnoreWords):
             self.connection.mass_action(cl)
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
 
-class AddSceneNumberingToTvEpisodes(AddSportsOption):
+# 30 -> 31
-    def test(self):
+class AddSceneNumberingToTvEpisodes(db.SchemaUpgrade):
-        return self.checkDBVersion() >= 31
-
     def execute(self):
-        backupDatabase(31)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column scene_season and scene_episode to tvepisodes")
         self.addColumn("tv_episodes", "scene_season", "NUMERIC", "NULL")
         self.addColumn("tv_episodes", "scene_episode", "NUMERIC", "NULL")
 
         self.incDBVersion()
+        return self.incDBVersion()
 
-class AddAnimeTVShow(AddSceneNumberingToTvEpisodes):
-    def test(self):
-        return self.checkDBVersion() >= 32
 
+# 31 -> 32
+class AddAnimeTVShow(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(32)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column anime to tv_episodes")
         self.addColumn("tv_shows", "anime", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddAbsoluteNumbering(AddAnimeTVShow):
-    def test(self):
-        return self.checkDBVersion() >= 33
 
+# 32 -> 33
+class AddAbsoluteNumbering(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(33)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column absolute_number to tv_episodes")
         self.addColumn("tv_episodes", "absolute_number", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddSceneAbsoluteNumbering(AddAbsoluteNumbering):
-    def test(self):
-        return self.checkDBVersion() >= 34
 
+# 33 -> 34
+class AddSceneAbsoluteNumbering(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(34)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column absolute_number and scene_absolute_number to scene_numbering")
         self.addColumn("scene_numbering", "absolute_number", "NUMERIC", "0")
         self.addColumn("scene_numbering", "scene_absolute_number", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddAnimeBlacklistWhitelist(AddSceneAbsoluteNumbering):
-
-    def test(self):
-        return self.checkDBVersion() >= 35
 
+# 34 -> 35
+class AddAnimeBlacklistWhitelist(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(35)
+        backupDatabase(self.checkDBVersion())
 
         cl = []
         cl.append(["CREATE TABLE blacklist (show_id INTEGER, range TEXT, keyword TEXT)"])
@@ -848,50 +842,50 @@ class AddAnimeBlacklistWhitelist(AddSceneAbsoluteNumbering):
         self.connection.mass_action(cl)
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddSceneAbsoluteNumbering(AddAnimeBlacklistWhitelist):
-    def test(self):
-        return self.checkDBVersion() >= 36
 
+# 35 -> 36
+class AddSceneAbsoluteNumbering2(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(36)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column scene_absolute_number to tv_episodes")
         self.addColumn("tv_episodes", "scene_absolute_number", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddXemRefresh(AddSceneAbsoluteNumbering):
-    def test(self):
-        return self.checkDBVersion() >= 37
 
+# 36 -> 37
+class AddXemRefresh(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(37)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Creating table xem_refresh")
         self.connection.action(
             "CREATE TABLE xem_refresh (indexer TEXT, indexer_id INTEGER PRIMARY KEY, last_refreshed INTEGER)")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddSceneToTvShows(AddXemRefresh):
-    def test(self):
-        return self.checkDBVersion() >= 38
 
+# 37 -> 38
+class AddSceneToTvShows(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(38)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column scene to tv_shows")
         self.addColumn("tv_shows", "scene", "NUMERIC", "0")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddIndexerMapping(AddSceneToTvShows):
-    def test(self):
-        return self.checkDBVersion() >= 39
 
+# 38 -> 39
+class AddIndexerMapping(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(39)
+        backupDatabase(self.checkDBVersion())
 
         if self.hasTable("indexer_mapping"):
             self.connection.action("DROP TABLE indexer_mapping")
@@ -901,13 +895,13 @@ class AddIndexerMapping(AddSceneToTvShows):
             "CREATE TABLE indexer_mapping (indexer_id INTEGER, indexer NUMERIC, mindexer_id INTEGER, mindexer NUMERIC, PRIMARY KEY (indexer_id, indexer))")
 
         self.incDBVersion()
+        return self.checkDBVersion()
 
-class AddVersionToTvEpisodes(AddIndexerMapping):
-    def test(self):
-        return self.checkDBVersion() >= 40
 
+# 39 -> 40
+class AddVersionToTvEpisodes(db.SchemaUpgrade):
     def execute(self):
-        backupDatabase(40)
+        backupDatabase(self.checkDBVersion())
 
         logger.log(u"Adding column version to tv_episodes and history")
         self.addColumn("tv_episodes", "version", "NUMERIC", "-1")
@@ -915,3 +909,46 @@ class AddVersionToTvEpisodes(AddIndexerMapping):
         self.addColumn("history", "version", "NUMERIC", "-1")
 
         self.incDBVersion()
+        return self.checkDBVersion()
+
+
+# 40 -> 10000
+class BumpDatabaseVersion(db.SchemaUpgrade):
+    def execute(self):
+        backupDatabase(self.checkDBVersion())
+        logger.log(u'Bumping database version')
+
+        self.setDBVersion(10000)
+        return self.checkDBVersion()
+
+# 41 -> 10001
+class Migrate41(db.SchemaUpgrade):
+    def execute(self):
+        backupDatabase(self.checkDBVersion())
+
+        logger.log(u'Bumping database version')
+
+        self.setDBVersion(10001)
+        return self.checkDBVersion()
+
+
+# 10000 -> 20000
+class SickGearDatabaseVersion(db.SchemaUpgrade):
+    def execute(self):
+        backupDatabase(self.checkDBVersion())
+
+        logger.log('Bumping database version to new SickGear standards')
+
+        self.setDBVersion(20000)
+        return self.checkDBVersion()
+
+# 10001 -> 10000
+class RemoveDefaultEpStatusFromTvShows(db.SchemaUpgrade):
+    def execute(self):
+        backupDatabase(self.checkDBVersion())
+
+        logger.log(u'Dropping column default_ep_status from tv_shows')
+        self.dropColumn('tv_shows', 'default_ep_status')
+
+        self.setDBVersion(10000)
+        return self.checkDBVersion()

sickbeard/db.py

@@ -356,6 +356,69 @@ class SchemaUpgrade(object):
         self.connection.action("ALTER TABLE %s ADD %s %s" % (table, column, type))
         self.connection.action("UPDATE %s SET %s = ?" % (table, column), (default,))
 
+    def dropColumn(self, table, column):
+        # get old table columns and store the ones we want to keep
+        result = self.connection.select('pragma table_info(%s)' % table)
+        keptColumns = [c for c in result if c['name'] != column]
+
+        keptColumnsNames = []
+        final = []
+        pk = []
+
+        # copy the old table schema, column by column
+        for column in keptColumns:
+
+            keptColumnsNames.append(column['name'])
+
+            cl = []
+            cl.append(column['name'])
+            cl.append(column['type'])
+
+            '''
+            To be implemented if ever required
+            if column['dflt_value']:
+                cl.append(str(column['dflt_value']))
+
+            if column['notnull']:
+                cl.append(column['notnull'])
+            '''
+
+            if int(column['pk']) != 0:
+                pk.append(column['name'])
+
+            b = ' '.join(cl)
+            final.append(b)
+
+        # join all the table column creation fields
+        final = ', '.join(final)
+        keptColumnsNames = ', '.join(keptColumnsNames)
+
+        # generate sql for the new table creation
+        if len(pk) == 0:
+            sql = 'CREATE TABLE %s_new (%s)' % (table, final)
+        else:
+            pk = ', '.join(pk)
+            sql = 'CREATE TABLE %s_new (%s, PRIMARY KEY(%s))' % (table, final, pk)
+
+        # create new temporary table and copy the old table data across, barring the removed column
+        self.connection.action(sql)
+        self.connection.action('INSERT INTO %s_new SELECT %s FROM %s' % (table, keptColumnsNames, table))
+
+        # copy the old indexes from the old table
+        result = self.connection.select('SELECT sql FROM sqlite_master WHERE tbl_name=? and type="index"', [table])
+
+        # remove the old table and rename the new table to take it's place
+        self.connection.action('DROP TABLE %s' % table)
+        self.connection.action('ALTER TABLE %s_new RENAME TO %s' % (table, table))
+
+        # write any indexes to the new table
+        if len(result) > 0:
+            for index in result:
+                self.connection.action(index['sql'])
+
+        # vacuum the db as we will have a lot of space to reclaim after dropping tables
+        self.connection.action("VACUUM")
+
     def checkDBVersion(self):
         return self.connection.checkDBVersion()
 
@@ -363,3 +426,82 @@ class SchemaUpgrade(object):
         new_version = self.checkDBVersion() + 1
         self.connection.action("UPDATE db_version SET db_version = ?", [new_version])
         return new_version
+
+    def setDBVersion(self, new_version):
+        self.connection.action("UPDATE db_version SET db_version = ?", [new_version])
+        return new_version
+
+
+def MigrationCode(myDB):
+
+    schema = {
+        0: sickbeard.mainDB.InitialSchema,  # 0->20000
+        9: sickbeard.mainDB.AddSizeAndSceneNameFields,
+        10: sickbeard.mainDB.RenameSeasonFolders,
+        11: sickbeard.mainDB.Add1080pAndRawHDQualities,
+        12: sickbeard.mainDB.AddShowidTvdbidIndex,
+        13: sickbeard.mainDB.AddLastUpdateTVDB,
+        14: sickbeard.mainDB.AddDBIncreaseTo15,
+        15: sickbeard.mainDB.AddIMDbInfo,
+        16: sickbeard.mainDB.AddProperNamingSupport,
+        17: sickbeard.mainDB.AddEmailSubscriptionTable,
+        18: sickbeard.mainDB.AddProperSearch,
+        19: sickbeard.mainDB.AddDvdOrderOption,
+        20: sickbeard.mainDB.AddSubtitlesSupport,
+        21: sickbeard.mainDB.ConvertTVShowsToIndexerScheme,
+        22: sickbeard.mainDB.ConvertTVEpisodesToIndexerScheme,
+        23: sickbeard.mainDB.ConvertIMDBInfoToIndexerScheme,
+        24: sickbeard.mainDB.ConvertInfoToIndexerScheme,
+        25: sickbeard.mainDB.AddArchiveFirstMatchOption,
+        26: sickbeard.mainDB.AddSceneNumbering,
+        27: sickbeard.mainDB.ConvertIndexerToInteger,
+        28: sickbeard.mainDB.AddRequireAndIgnoreWords,
+        29: sickbeard.mainDB.AddSportsOption,
+        30: sickbeard.mainDB.AddSceneNumberingToTvEpisodes,
+        31: sickbeard.mainDB.AddAnimeTVShow,
+        32: sickbeard.mainDB.AddAbsoluteNumbering,
+        33: sickbeard.mainDB.AddSceneAbsoluteNumbering,
+        34: sickbeard.mainDB.AddAnimeBlacklistWhitelist,
+        35: sickbeard.mainDB.AddSceneAbsoluteNumbering2,
+        36: sickbeard.mainDB.AddXemRefresh,
+        37: sickbeard.mainDB.AddSceneToTvShows,
+        38: sickbeard.mainDB.AddIndexerMapping,
+        39: sickbeard.mainDB.AddVersionToTvEpisodes,
+
+        40: sickbeard.mainDB.BumpDatabaseVersion,
+        41: sickbeard.mainDB.Migrate41,
+
+        10000: sickbeard.mainDB.SickGearDatabaseVersion,
+        10001: sickbeard.mainDB.RemoveDefaultEpStatusFromTvShows
+
+        #20000: sickbeard.mainDB.AddCoolSickGearFeature1,
+        #20001: sickbeard.mainDB.AddCoolSickGearFeature2,
+        #20002: sickbeard.mainDB.AddCoolSickGearFeature3,
+    }
+
+    db_version = myDB.checkDBVersion()
+    logger.log(u'Detected database version: v' + str(db_version), logger.DEBUG)
+
+    if not (db_version in schema):
+        if db_version == sickbeard.mainDB.MAX_DB_VERSION:
+            logger.log(u'Database schema is up-to-date, no upgrade required')
+        elif db_version < 10000:
+            logger.log_error_and_exit(u'SickGear does not currently support upgrading from this database version')
+        else:
+            logger.log_error_and_exit(u'Invalid database version')
+
+    else:
+
+        while db_version < sickbeard.mainDB.MAX_DB_VERSION:
+            try:
+                update = schema[db_version](myDB)
+                db_version = update.execute()
+            except Exception, e:
+                myDB.close()
+                logger.log(u'Failed to update database with error: ' + ex(e) + ' attempting recovery...', logger.ERROR)
+
+                if restoreDatabase(db_version):
+                    # initialize the main SB database
+                    logger.log_error_and_exit(u'Successfully restored database version:' + str(db_version))
+                else:
+                    logger.log_error_and_exit(u'Failed to restore database version:' + str(db_version))

tests/db_tests.py

@@ -29,7 +29,7 @@ class DBBasicTests(test.SickbeardTestDBCase):
 
     def test_select(self):
         self.db.select("SELECT * FROM tv_episodes WHERE showid = ? AND location != ''", [0000])
-
+        self.db.close()
 
 if __name__ == '__main__':
     print "=================="

tests/migration_tests.py

@@ -0,0 +1,68 @@
+import sys
+import os.path
+import glob
+import unittest
+import test_lib as test
+import sickbeard
+from time import sleep
+from sickbeard import db
+
+sys.path.append(os.path.abspath('..'))
+sys.path.append(os.path.abspath('../lib'))
+
+sickbeard.SYS_ENCODING = 'UTF-8'
+
+
+class MigrationBasicTests(test.SickbeardTestDBCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_migrations(self):
+        schema = {
+                    0: sickbeard.mainDB.InitialSchema,
+                    31: sickbeard.mainDB.AddAnimeTVShow,
+                    32: sickbeard.mainDB.AddAbsoluteNumbering,
+                    33: sickbeard.mainDB.AddSceneAbsoluteNumbering,
+                    34: sickbeard.mainDB.AddAnimeBlacklistWhitelist,
+                    35: sickbeard.mainDB.AddSceneAbsoluteNumbering2,
+                    36: sickbeard.mainDB.AddXemRefresh,
+                    37: sickbeard.mainDB.AddSceneToTvShows,
+                    38: sickbeard.mainDB.AddIndexerMapping,
+                    39: sickbeard.mainDB.AddVersionToTvEpisodes,
+                    41: AddDefaultEpStatusToTvShows,
+                }
+
+        count = 1
+        while count < len(schema.keys()):
+            myDB = db.DBConnection()
+
+            for version in sorted(schema.keys())[:count]:
+                update = schema[version](myDB)
+                update.execute()
+                sleep(0.1)
+
+            db.MigrationCode(myDB)
+            myDB.close()
+            for filename in glob.glob(os.path.join(test.TESTDIR, test.TESTDBNAME) +'*'):
+                os.remove(filename)
+
+            sleep(0.1)
+            count += 1
+
+
+class AddDefaultEpStatusToTvShows(db.SchemaUpgrade):
+    def execute(self):
+        self.addColumn("tv_shows", "default_ep_status", "TEXT", "")
+        self.setDBVersion(41)
+
+
+if __name__ == '__main__':
+    print "=================="
+    print "STARTING - MIGRATION TESTS"
+    print "=================="
+    print "######################################################################"
+    suite = unittest.TestLoader().loadTestsFromTestCase(MigrationBasicTests)
+    unittest.TextTestRunner(verbosity=2).run(suite)

tests/test_lib.py

@@ -20,11 +20,11 @@
 from __future__ import with_statement
 
 import unittest
-
 import sqlite3
-
+import glob
 import sys
 import os.path
+
 sys.path.append(os.path.abspath('..'))
 sys.path.append(os.path.abspath('../lib'))
 
@@ -173,7 +173,7 @@ def setUp_test_db():
     """upgrades the db to the latest version
     """
     # upgrading the db
-    db.upgradeDatabase(db.DBConnection(), mainDB.InitialSchema)
+    db.MigrationCode(db.DBConnection())
 
     # fix up any db problems
     db.sanityCheckDatabase(db.DBConnection(), mainDB.MainSanityCheck)
@@ -191,8 +191,9 @@ def tearDown_test_db():
     """
     # uncomment next line so leave the db intact between test and at the end
     #return False
-    if os.path.exists(os.path.join(TESTDIR, TESTDBNAME)):
+
-        os.remove(os.path.join(TESTDIR, TESTDBNAME))
+    for filename in glob.glob(os.path.join(TESTDIR, TESTDBNAME) + '*'):
+        os.remove(filename)
     if os.path.exists(os.path.join(TESTDIR, TESTCACHEDBNAME)):
         os.remove(os.path.join(TESTDIR, TESTCACHEDBNAME))
     if os.path.exists(os.path.join(TESTDIR, TESTFAILEDDBNAME)):