Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions django/db/backends/base/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class BaseDatabaseFeatures:
can_use_chunked_reads = True
can_return_columns_from_insert = False
can_return_rows_from_bulk_insert = False
can_return_rows_from_update = False
has_bulk_insert = True
uses_savepoints = True
can_release_savepoints = False
Expand Down
1 change: 1 addition & 0 deletions django/db/backends/oracle/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ def __init__(self, *args, **kwargs):
"use_returning_into", True
)
self.features.can_return_columns_from_insert = use_returning_into
self.features.can_return_rows_from_update = use_returning_into

@property
def is_pool(self):
Expand Down
1 change: 1 addition & 0 deletions django/db/backends/oracle/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
has_select_for_update_of = True
select_for_update_of_column = True
can_return_columns_from_insert = True
can_return_rows_from_update = True
supports_subqueries_in_group_by = False
ignores_unnecessary_order_by_in_subqueries = False
supports_tuple_comparison_against_subquery = False
Expand Down
1 change: 1 addition & 0 deletions django/db/backends/postgresql/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
allows_group_by_selected_pks = True
can_return_columns_from_insert = True
can_return_rows_from_bulk_insert = True
can_return_rows_from_update = True
has_real_datatype = True
has_native_uuid_field = True
has_native_duration_field = True
Expand Down
4 changes: 4 additions & 0 deletions django/db/backends/sqlite3/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,7 @@ def can_return_columns_from_insert(self):
can_return_rows_from_bulk_insert = property(
operator.attrgetter("can_return_columns_from_insert")
)

can_return_rows_from_update = property(
operator.attrgetter("can_return_columns_from_insert")
)
93 changes: 70 additions & 23 deletions django/db/models/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1094,12 +1094,33 @@ def _save_table(
]
forced_update = update_fields or force_update
pk_val = self._get_pk_val(meta)
updated = self._do_update(
base_qs, using, pk_val, values, update_fields, forced_update
returning_fields = [
f
for f in meta.local_concrete_fields
if (
f.generated
and f.referenced_fields.intersection(non_pks_non_generated)
)
]
for field, _model, value in values:
if (update_fields is None or field.name in update_fields) and hasattr(
value, "resolve_expression"
):
returning_fields.append(field)
results = self._do_update(
base_qs,
using,
pk_val,
values,
update_fields,
forced_update,
returning_fields,
)
if force_update and not updated:
if updated := bool(results):
self._assign_returned_values(results[0], returning_fields)
elif force_update:
raise self.NotUpdated("Forced update did not affect any rows.")
if update_fields and not updated:
elif update_fields:
raise self.NotUpdated(
"Save with update_fields did not affect any rows."
)
Expand All @@ -1126,16 +1147,32 @@ def _save_table(
for f in meta.local_concrete_fields
if not f.generated and (pk_set or f is not meta.auto_field)
]
returning_fields = meta.db_returning_fields
returning_fields = list(meta.db_returning_fields)
for field in fields:
value = (
getattr(self, field.attname) if raw else field.pre_save(self, False)
)
if hasattr(value, "resolve_expression"):
returning_fields.append(field)
elif field.db_returning:
returning_fields.remove(field)
Comment on lines +1157 to +1158
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If an explicit literal value is provided for a field there is no need to return it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Except that I've written some code like this for MongodB where the id is ObjectIdAutoField: Question.objects.create(id="000000000000000000000013"). Previously, this would refresh id to the returned value, ObjectId("000000000000000000000013"). What do you think?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess the same problem might happen with some form of UUID field that returns uuid.UUID instances I guess. Could you write a regression test for it and file an issue?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I imagine UUIDAutoField (ticket-32577) could reproduce it. Do you have a more generic way in mind? I could try for some minimal implementation of UUIDFIeld + django.contrib.postgres.functions.RandomUUID to create a regression test in postgres_tests but this might be unnecessarily complicated in light of the MongoDB tests.

This works as a regression test, at least on SQLite and PostgreSQL. I'm unsure if all databases will accept it.

a = AutoModel.objects.create(pk="1")
self.assertEqual(a.pk, 1)

Copy link
Member Author

@charettes charettes Sep 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This works as a regression test, at least on SQLite and PostgreSQL. I'm unsure if all databases will accept it.

That's perfect, the whole notion of auto-field is problematic anyway as discussed on #18365 (as any field with a db_default and primary_key=True should be allowed as a primary key in the first place).

I'm surprised we don't already have a test that covers this use case though 🤔

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR for the regression: #19868

results = self._do_insert(
cls._base_manager, using, fields, returning_fields, raw
)
if results:
for value, field in zip(results[0], returning_fields):
setattr(self, field.attname, value)
self._assign_returned_values(results[0], returning_fields)
return updated

def _do_update(self, base_qs, using, pk_val, values, update_fields, forced_update):
def _do_update(
self,
base_qs,
using,
pk_val,
values,
update_fields,
forced_update,
returning_fields,
):
"""
Try to update the model. Return True if the model was updated (if an
update query was done and a matching row was found in the DB).
Expand All @@ -1147,22 +1184,23 @@ def _do_update(self, base_qs, using, pk_val, values, update_fields, forced_updat
# case we just say the update succeeded. Another case ending up
# here is a model with just PK - in that case check that the PK
# still exists.
return update_fields is not None or filtered.exists()
if update_fields is not None or filtered.exists():
return [()]
return []
if self._meta.select_on_save and not forced_update:
return (
filtered.exists()
and
# It may happen that the object is deleted from the DB right
# after this check, causing the subsequent UPDATE to return
# zero matching rows. The same result can occur in some rare
# cases when the database returns zero despite the UPDATE being
# executed successfully (a row is matched and updated). In
# order to distinguish these two cases, the object's existence
# in the database is again checked for if the UPDATE query
# returns 0.
(filtered._update(values) > 0 or filtered.exists())
)
return filtered._update(values) > 0
# It may happen that the object is deleted from the DB right after
# this check, causing the subsequent UPDATE to return zero matching
# rows. The same result can occur in some rare cases when the
# database returns zero despite the UPDATE being executed
# successfully (a row is matched and updated). In order to
# distinguish these two cases, the object's existence in the
# database is again checked for if the UPDATE query returns 0.
if not filtered.exists():
return []
if results := filtered._update(values, returning_fields):
return results
return [()] if filtered.exists() else []
return filtered._update(values, returning_fields)

def _do_insert(self, manager, using, fields, returning_fields, raw):
"""
Expand All @@ -1177,6 +1215,15 @@ def _do_insert(self, manager, using, fields, returning_fields, raw):
raw=raw,
)

def _assign_returned_values(self, returned_values, returning_fields):
returning_fields_iter = iter(returning_fields)
for value, field in zip(returned_values, returning_fields_iter):
setattr(self, field.attname, value)
# Defer all fields that were meant to be updated with their database
# resolved values but couldn't as they are effectively stale.
for field in returning_fields_iter:
self.__dict__.pop(field.attname, None)

def _prepare_related_fields_for_save(self, operation_name, fields=None):
# Ensure that a model instance without a PK hasn't been assigned to
# a ForeignKey, GenericForeignKey or OneToOneField on this model. If
Expand Down
10 changes: 10 additions & 0 deletions django/db/models/fields/generated.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,16 @@ def generated_sql(self, connection):
sql = f"CASE WHEN {sql} THEN 1 ELSE 0 END"
return sql, params

@cached_property
def referenced_fields(self):
resolved_expression = self.expression.resolve_expression(
self._query, allow_joins=False
)
referenced_fields = []
for col in self._query._gen_cols([resolved_expression]):
referenced_fields.append(col.target)
return frozenset(referenced_fields)

def check(self, **kwargs):
databases = kwargs.get("databases") or []
errors = [
Expand Down
6 changes: 4 additions & 2 deletions django/db/models/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -1306,7 +1306,7 @@ async def aupdate(self, **kwargs):

aupdate.alters_data = True

def _update(self, values):
def _update(self, values, returning_fields=None):
"""
A version of update() that accepts field objects instead of field
names. Used primarily for model saving and not intended for use by
Expand All @@ -1320,7 +1320,9 @@ def _update(self, values):
# Clear any annotations so that they won't be present in subqueries.
query.annotations = {}
self._result_cache = None
return query.get_compiler(self.db).execute_sql(ROW_COUNT)
if returning_fields is None:
return query.get_compiler(self.db).execute_sql(ROW_COUNT)
return query.get_compiler(self.db).execute_returning_sql(returning_fields)

_update.alters_data = True
_update.queryset_only = False
Expand Down
44 changes: 44 additions & 0 deletions django/db/models/sql/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2020,6 +2020,9 @@ def as_sql(self):


class SQLUpdateCompiler(SQLCompiler):
returning_fields = None
returning_params = ()

def as_sql(self):
"""
Create the SQL for this query. Return the SQL string and list of
Expand Down Expand Up @@ -2087,6 +2090,15 @@ def as_sql(self):
params = []
else:
result.append("WHERE %s" % where)
if self.returning_fields:
# Skip empty r_sql to allow subclasses to customize behavior for
# 3rd party backends. Refs #19096.
r_sql, self.returning_params = self.connection.ops.returning_columns(
self.returning_fields
)
if r_sql:
result.append(r_sql)
params.extend(self.returning_params)
return " ".join(result), tuple(update_params + params)

def execute_sql(self, result_type):
Expand All @@ -2110,6 +2122,38 @@ def execute_sql(self, result_type):
is_empty = False
return row_count

def execute_returning_sql(self, returning_fields):
"""
Execute the specified update and return rows of the returned columns
associated with the specified returning_field if the backend supports
it.
"""
if self.query.get_related_updates():
raise NotImplementedError(
"Update returning is not implemented for queries with related updates."
)

if (
not returning_fields
or not self.connection.features.can_return_rows_from_update
):
row_count = self.execute_sql(ROW_COUNT)
return [()] * row_count

self.returning_fields = returning_fields
with self.connection.cursor() as cursor:
sql, params = self.as_sql()
cursor.execute(sql, params)
rows = self.connection.ops.fetch_returned_rows(
cursor, self.returning_params
)
opts = self.query.get_meta()
cols = [field.get_col(opts.db_table) for field in self.returning_fields]
converters = self.get_converters(cols)
if converters:
rows = self.apply_converters(rows, converters)
return list(rows)

def pre_sql_setup(self):
"""
If the update depends on results from other tables, munge the "where"
Expand Down
39 changes: 17 additions & 22 deletions docs/ref/models/expressions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,6 @@ Some examples

# Create a new company using expressions.
>>> company = Company.objects.create(name="Google", ticker=Upper(Value("goog")))
# Be sure to refresh it if you need to access the field.
>>> company.refresh_from_db()
>>> company.ticker
'GOOG'

Expand Down Expand Up @@ -157,12 +155,6 @@ know about it - it is dealt with entirely by the database. All Python does,
through Django's ``F()`` class, is create the SQL syntax to refer to the field
and describe the operation.

To access the new value saved this way, the object must be reloaded::

reporter = Reporters.objects.get(pk=reporter.pk)
# Or, more succinctly:
reporter.refresh_from_db()

As well as being used in operations on single instances as above, ``F()`` can
be used with ``update()`` to perform bulk updates on a ``QuerySet``. This
reduces the two queries we were using above - the ``get()`` and the
Expand Down Expand Up @@ -199,7 +191,6 @@ array-slicing syntax. The indices are 0-based and the ``step`` argument to
>>> writer = Writers.objects.get(name="Priyansh")
>>> writer.name = F("name")[1:5]
>>> writer.save()
>>> writer.refresh_from_db()
>>> writer.name
'riya'

Expand All @@ -221,23 +212,27 @@ robust: it will only ever update the field based on the value of the field in
the database when the :meth:`~Model.save` or ``update()`` is executed, rather
than based on its value when the instance was retrieved.

``F()`` assignments persist after ``Model.save()``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``F()`` assignments are refreshed after ``Model.save()``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

``F()`` objects assigned to model fields persist after saving the model
instance and will be applied on each :meth:`~Model.save`. For example::
``F()`` objects assigned to model fields are refreshed from the database on
:meth:`~Model.save` on backends that support it without incurring a subsequent
query (SQLite, PostgreSQL, and Oracle) and deferred otherwise (MySQL or
MariaDB). For example:

reporter = Reporters.objects.get(name="Tintin")
reporter.stories_filed = F("stories_filed") + 1
reporter.save()
.. code-block:: pycon

reporter.name = "Tintin Jr."
reporter.save()
>>> reporter = Reporters.objects.get(name="Tintin")
>>> reporter.stories_filed = F("stories_filed") + 1
>>> reporter.save()
>>> reporter.stories_filed # This triggers a refresh query on MySQL/MariaDB.
14 # Assuming the database value was 13 when the object was saved.

.. versionchanged:: 6.0

``stories_filed`` will be updated twice in this case. If it's initially ``1``,
the final value will be ``3``. This persistence can be avoided by reloading the
model object after saving it, for example, by using
:meth:`~Model.refresh_from_db`.
In previous versions of Django, ``F()`` objects were not refreshed from the
database on :meth:`~Model.save` which resulted in them being evaluated and
persisted every time the instance was saved.

Using ``F()`` in filters
~~~~~~~~~~~~~~~~~~~~~~~~
Expand Down
12 changes: 6 additions & 6 deletions docs/ref/models/fields.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1315,12 +1315,6 @@ materialized view.
PostgreSQL only supports persisted columns. Oracle only supports virtual
columns.

.. admonition:: Refresh the data

Since the database computes the value, the object must be reloaded to
access the new value after :meth:`~Model.save`, for example, by using
:meth:`~Model.refresh_from_db`.

.. admonition:: Database limitations

There are many database-specific restrictions on generated fields that
Expand All @@ -1338,6 +1332,12 @@ materialized view.
.. _PostgreSQL: https://www.postgresql.org/docs/current/ddl-generated-columns.html
.. _SQLite: https://www.sqlite.org/gencol.html#limitations

.. versionchanged:: 6.0

``GeneratedField``\s are now automatically refreshed from the database on
backends that support it (SQLite, PostgreSQL, and Oracle) and marked as
deferred otherwise.

``GenericIPAddressField``
-------------------------

Expand Down
10 changes: 10 additions & 0 deletions docs/releases/6.0.txt
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,13 @@ Models
value from the non-null input values. This is supported on SQLite, MySQL,
Oracle, and PostgreSQL 16+.

* :class:`~django.db.models.GeneratedField`\s and :ref:`fields assigned
expressions <avoiding-race-conditions-using-f>` are now refreshed from the
database after :meth:`~django.db.models.Model.save` on backends that support
the ``RETURNING`` clause (SQLite, PostgreSQL, and Oracle). On backends that
don't support it (MySQL and MariaDB), the fields are marked as deferred to
trigger a refresh on subsequent accesses.

Pagination
~~~~~~~~~~

Expand Down Expand Up @@ -420,6 +427,9 @@ backends.
``returning_params`` to be provided just like
``fetch_returned_insert_columns()`` did.

* If the database supports ``UPDATE … RETURNING`` statements, backends can set
``DatabaseFeatures.can_return_rows_from_update=True``.

Dropped support for MariaDB 10.5
--------------------------------

Expand Down
Loading
Loading