The search backend should silently fail on SQL schema issues when running in a migration #21747
Description
NetBox Edition
NetBox Community
NetBox Version
v4.5.5
Python Version
3.12
Steps to Reproduce
This issue surfaced when introducing a new field on the CustomField model under #19025. If the state of the database doesn't match the model's state when an object is created, updated, or deleted, calling cache() on the search backend class can yield a ProgrammingError.
Expected Behavior
All migrations should complete successfully.
Observed Behavior
An exception is raised during the application of dcim.0206_load_module_type_profiles:
Traceback (most recent call last):
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/backends/utils.py", line 105, in _execute
return self.cursor.execute(sql, params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/psycopg/cursor.py", line 117, in execute
raise ex.with_traceback(None)
psycopg.errors.UndefinedColumn: column extras_customfield.validation_schema does not exist
LINE 1: ...aximum", "extras_customfield"."validation_regex", "extras_cu...
^
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/jstretch/projects/netbox/netbox/./manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/__init__.py", line 442, in execute_from_command_line
utility.execute()
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/__init__.py", line 436, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/commands/test.py", line 24, in run_from_argv
super().run_from_argv(argv)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/base.py", line 420, in run_from_argv
self.execute(*args, **cmd_options)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/base.py", line 464, in execute
output = self.handle(*args, **options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/commands/test.py", line 63, in handle
failures = test_runner.run_tests(test_labels)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/test/runner.py", line 1092, in run_tests
old_config = self.setup_databases(
^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/test/runner.py", line 990, in setup_databases
return _setup_databases(
^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/test/utils.py", line 204, in setup_databases
connection.creation.create_test_db(
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/backends/base/creation.py", line 78, in create_test_db
call_command(
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/__init__.py", line 194, in call_command
return command.execute(*args, **defaults)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/base.py", line 464, in execute
output = self.handle(*args, **options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/base.py", line 111, in wrapper
res = handle_func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/core/management/commands/migrate.py", line 353, in handle
post_migrate_state = executor.migrate(
^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/migrations/executor.py", line 135, in migrate
state = self._migrate_all_forwards(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/migrations/executor.py", line 167, in _migrate_all_forwards
state = self.apply_migration(
^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/migrations/executor.py", line 255, in apply_migration
state = migration.apply(state, schema_editor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/migrations/migration.py", line 132, in apply
operation.database_forwards(
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/migrations/operations/special.py", line 196, in database_forwards
self.code(from_state.apps, schema_editor)
File "/home/jstretch/projects/netbox/netbox/dcim/migrations/0206_load_module_type_profiles.py", line 34, in load_initial_data
raise e
File "/home/jstretch/projects/netbox/netbox/dcim/migrations/0206_load_module_type_profiles.py", line 31, in load_initial_data
ModuleTypeProfile.objects.using(db_alias).create(**data)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/query.py", line 665, in create
obj.save(force_insert=True, using=self.db)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/base.py", line 902, in save
self.save_base(
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/base.py", line 1023, in save_base
post_save.send(
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/dispatch/dispatcher.py", line 189, in send
response = receiver(signal=self, sender=sender, **named)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/projects/netbox/netbox/netbox/search/backends.py", line 66, in caching_handler
self.cache(instance, remove_existing=not created)
File "/home/jstretch/projects/netbox/netbox/netbox/search/backends.py", line 213, in cache
cf for cf in CustomField.objects.get_for_model(indexer.model)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/query.py", line 386, in __iter__
self._fetch_all()
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/query.py", line 1954, in _fetch_all
self._result_cache = list(self._iterable_class(self))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/query.py", line 93, in __iter__
results = compiler.execute_sql(
^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/models/sql/compiler.py", line 1623, in execute_sql
cursor.execute(sql, params)
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/backends/utils.py", line 79, in execute
return self._execute_with_wrappers(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/backends/utils.py", line 92, in _execute_with_wrappers
return executor(sql, params, many, context)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/backends/utils.py", line 100, in _execute
with self.db.wrap_database_errors:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/utils.py", line 91, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/django/db/backends/utils.py", line 105, in _execute
return self.cursor.execute(sql, params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jstretch/.venv/netbox/lib/python3.12/site-packages/psycopg/cursor.py", line 117, in execute
raise ex.with_traceback(None)
django.db.utils.ProgrammingError: column extras_customfield.validation_schema does not exist
LINE 1: ...aximum", "extras_customfield"."validation_regex", "extras_cu...
^