119 lines
5.3 KiB
Python
119 lines
5.3 KiB
Python
# Generated by Django 2.2.8 on 2020-02-21 16:31
|
|
|
|
from django.db import migrations, models, connection
|
|
|
|
|
|
def migrate_event_data(apps, schema_editor):
|
|
# see: https://github.com/ansible/awx/issues/6010
|
|
#
|
|
# the goal of this function is to end with event tables (e.g., main_jobevent)
|
|
# that have a bigint primary key (because the old usage of an integer
|
|
# numeric isn't enough, as its range is about 2.1B, see:
|
|
# https://www.postgresql.org/docs/9.1/datatype-numeric.html)
|
|
|
|
# unfortunately, we can't do this with a simple ALTER TABLE, because
|
|
# for tables with hundreds of millions or billions of rows, the ALTER TABLE
|
|
# can take *hours* on modest hardware.
|
|
#
|
|
# the approach in this migration means that post-migration, event data will
|
|
# *not* immediately show up, but will be repopulated over time progressively
|
|
# the trade-off here is not having to wait hours for the full data migration
|
|
# before you can start and run AWX again (including new playbook runs)
|
|
for tblname in (
|
|
'main_jobevent', 'main_inventoryupdateevent',
|
|
'main_projectupdateevent', 'main_adhoccommandevent',
|
|
'main_systemjobevent'
|
|
):
|
|
with connection.cursor() as cursor:
|
|
# rename the current event table
|
|
cursor.execute(
|
|
f'ALTER TABLE {tblname} RENAME TO _old_{tblname};'
|
|
)
|
|
# create a *new* table with the same schema
|
|
cursor.execute(
|
|
f'CREATE TABLE {tblname} (LIKE _old_{tblname} INCLUDING ALL);'
|
|
)
|
|
# alter the *new* table so that the primary key is a big int
|
|
cursor.execute(
|
|
f'ALTER TABLE {tblname} ALTER COLUMN id TYPE bigint USING id::bigint;'
|
|
)
|
|
|
|
# recreate counter for the new table's primary key to
|
|
# start where the *old* table left off (we have to do this because the
|
|
# counter changed from an int to a bigint)
|
|
cursor.execute(f'DROP SEQUENCE IF EXISTS "{tblname}_id_seq" CASCADE;')
|
|
cursor.execute(f'CREATE SEQUENCE "{tblname}_id_seq";')
|
|
cursor.execute(
|
|
f'ALTER TABLE "{tblname}" ALTER COLUMN "id" '
|
|
f"SET DEFAULT nextval('{tblname}_id_seq');"
|
|
)
|
|
cursor.execute(
|
|
f"SELECT setval('{tblname}_id_seq', (SELECT MAX(id) FROM _old_{tblname}), true);"
|
|
)
|
|
|
|
# replace the BTREE index on main_jobevent.job_id with
|
|
# a BRIN index to drastically improve per-UJ lookup performance
|
|
# see: https://info.crunchydata.com/blog/postgresql-brin-indexes-big-data-performance-with-minimal-storage
|
|
if tblname == 'main_jobevent':
|
|
cursor.execute("SELECT indexname FROM pg_indexes WHERE tablename='main_jobevent' AND indexdef LIKE '%USING btree (job_id)';")
|
|
old_index = cursor.fetchone()[0]
|
|
cursor.execute(f'DROP INDEX {old_index}')
|
|
cursor.execute('CREATE INDEX main_jobevent_job_id_brin_idx ON main_jobevent USING brin (job_id);')
|
|
|
|
# remove all of the indexes and constraints from the old table
|
|
# (they just slow down the data migration)
|
|
cursor.execute(f"SELECT indexname, indexdef FROM pg_indexes WHERE tablename='_old_{tblname}' AND indexname != '{tblname}_pkey';")
|
|
indexes = cursor.fetchall()
|
|
|
|
cursor.execute(f"SELECT conname, contype, pg_catalog.pg_get_constraintdef(r.oid, true) as condef FROM pg_catalog.pg_constraint r WHERE r.conrelid = '_old_{tblname}'::regclass AND conname != '{tblname}_pkey';")
|
|
constraints = cursor.fetchall()
|
|
|
|
for indexname, indexdef in indexes:
|
|
cursor.execute(f'DROP INDEX IF EXISTS {indexname}')
|
|
for conname, contype, condef in constraints:
|
|
cursor.execute(f'ALTER TABLE _old_{tblname} DROP CONSTRAINT IF EXISTS {conname}')
|
|
|
|
|
|
class FakeAlterField(migrations.AlterField):
|
|
|
|
def database_forwards(self, *args):
|
|
# this is intentionally left blank, because we're
|
|
# going to accomplish the migration with some custom raw SQL
|
|
pass
|
|
|
|
|
|
class Migration(migrations.Migration):
|
|
|
|
dependencies = [
|
|
('main', '0112_v370_workflow_node_identifier'),
|
|
]
|
|
|
|
operations = [
|
|
migrations.RunPython(migrate_event_data),
|
|
FakeAlterField(
|
|
model_name='adhoccommandevent',
|
|
name='id',
|
|
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
|
),
|
|
FakeAlterField(
|
|
model_name='inventoryupdateevent',
|
|
name='id',
|
|
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
|
),
|
|
FakeAlterField(
|
|
model_name='jobevent',
|
|
name='id',
|
|
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
|
),
|
|
FakeAlterField(
|
|
model_name='projectupdateevent',
|
|
name='id',
|
|
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
|
),
|
|
FakeAlterField(
|
|
model_name='systemjobevent',
|
|
name='id',
|
|
field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
|
),
|
|
]
|