Django Database Migrations with Custom Operations

Snippet

Django Database Migrations with Custom Operations

Django migrations support custom operations for complex data transformations that go beyond simple field changes. This pattern is essential when you need to migrate data between tables, perform batch processing during migrations, or implement zero-downtime migration strategies. The MoveDataToNewTable operation demonstrates how to iterate through large datasets in batches, perform custom logic on each record, and maintain database consistency with atomic transactions.

snippet.py
python
from django.db import migrations
 
class MoveDataToNewTable(migrations.Operation):
    atomic = True
    
    def __init__(self, model_name, field_name, batch_size=1000):
        self.model_name = model_name
        self.field_name = field_name
        self.batch_size = batch_size
    
    def state_forwards(self, app_label, state):
        pass
    
    def database_forwards(self, app_label, schema_editor, from_state, to_state):
        model = state.get_model(app_label, self.model_name)
        field = model._meta.get_field(self.field_name)
        
        queryset = model.objects.filter(**{f'{self.field_name}__isnull': False})
        total = queryset.count()
        
        for offset in range(0, total, self.batch_size):
            batch = list(queryset[offset:offset + self.batch_size])
            for obj in batch:
                field.save(obj, obj)
            schema_editor.execute('SELECT 1')
    
    def describe(self):
        return f'Migrating {self.field_name} data in {self.model_name}'
 
class Migration(migrations.Migration):
    dependencies = [('myapp', '0001_initial')]
    
    operations = [
        migrations.AddField(
            model_name='product',
            name='cached_slug',
            field=models.SlugField(max_length=150, null=True),
        ),
        MoveDataToNewTable('product', 'cached_slug'),
        migrations.AlterField(
            model_name='product',
            name='cached_slug',
            field=models.SlugField(max_length=150),
        ),
    ]

django

Breakdown

class MoveDataToNewTable(migrations.Operation):

Custom migration operation class inheriting from migrations.Operation

atomic = True

Class attribute ensuring operation runs in a transaction

def database_forwards(self, app_label, schema_editor, from_state, to_state):

Main method called during migration execution

queryset.filter(**{f'{self.field_name}__isnull': False})

Dynamic field filtering using dictionary unpacking

for offset in range(0, total, self.batch_size):

Batch iteration pattern to process large datasets memory-efficiently

schema_editor.execute('SELECT 1')

Dummy query to trigger batch commit in PostgreSQL/MySQL

Previous snippet Next snippet

More Expert Python exercises →

From your library

Django Database Migrations with Custom Operations

Related