Skip to content

gh-117953: Track Extra Details in Global Extensions Cache #118532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ericsnowcurrently merged 24 commits into from
May 4, 2024
Merged

gh-117953: Track Extra Details in Global Extensions Cache #118532

Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
1fc1149
Store the cached def in a wrapping struct rather than directly.
ericsnowcurrently Apr 30, 2024
254e1f1
Pass the cached value around where appropriate.
ericsnowcurrently May 2, 2024
0694318
Pass the cached value to check_singlephase().
ericsnowcurrently May 1, 2024
9ca330b
Add extensions_cache_value helpers.
ericsnowcurrently Apr 30, 2024
7e0c4ee
Add get_cached_m_dict().
ericsnowcurrently Apr 30, 2024
c1c1379
Track ext module origin.
ericsnowcurrently May 1, 2024
8485fb5
Look up core module dicts in get_cached_m_dict().
ericsnowcurrently May 1, 2024
abbddde
Pass m_dict to _extensions_cache_set().
ericsnowcurrently May 1, 2024
12bcd96
Pass m_init to _extensions_cache_set().
ericsnowcurrently May 1, 2024
2d56ada
Add some comments.
ericsnowcurrently May 2, 2024
9c70d53
Use del_cached_m_dict() in set_cached_m_dict().
ericsnowcurrently May 2, 2024
0098057
Add m_init and m_dict fields to extensions_cache_value.
ericsnowcurrently May 1, 2024
57aaba2
Track which interpreter was used to create m_dict.
ericsnowcurrently May 1, 2024
96932eb
Track the interpreter ID.
ericsnowcurrently May 1, 2024
a4cb134
Look up core module dicts in get_cached_m_dict().
ericsnowcurrently May 1, 2024
c4d3be1
Make sure we only set m_dict for non-isolated interpreters.
ericsnowcurrently May 1, 2024
d87584d
Simplify.
ericsnowcurrently May 3, 2024
336d1da
Fix a comment.
ericsnowcurrently May 3, 2024
bd1e134
Separate the def logic from the rest of the extensions cache code.
ericsnowcurrently May 3, 2024
e42e797
Add the per-interpreter cache index to each global cache entry.
ericsnowcurrently May 3, 2024
faa4254
Fix an assert.
ericsnowcurrently May 4, 2024
5276981
Properly sync the cached m_index and the def m_index.
ericsnowcurrently May 3, 2024
f9a2a05
Merge branch 'main' into extensions-cache-better-tracking
ericsnowcurrently May 4, 2024
7d3dbc1
Fix make smelly.
ericsnowcurrently May 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Track the interpreter ID.
  • Loading branch information
@ericsnowcurrently
ericsnowcurrently committed May 3, 2024
commit 96932eb8601897a8283535de5cb46fcae3f54c5d
20 changes: 13 additions & 7 deletions Python/import.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -959,10 +959,12 @@ struct extensions_cache_value {
multiple initializations.
It is set by update_global_state_for_extension(). */
cached_m_dict_t m_dict;
PyInterpreterState *m_dict_interp;
int64_t m_dict_interpid;

_Py_ext_module_origin origin;
};
#define EXTENSIONS_CACHE_VALUE_INIT \
(struct extensions_cache_value){ .m_dict_interpid=-1 }

static struct extensions_cache_value *
alloc_extensions_cache_value(void)
Expand All @@ -973,7 +975,7 @@ alloc_extensions_cache_value(void)
PyErr_NoMemory();
return NULL;
}
*value = (struct extensions_cache_value){0};
*value = EXTENSIONS_CACHE_VALUE_INIT;
return value;
}

Expand All @@ -989,10 +991,9 @@ set_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict)
assert(value != NULL);
assert(value->def != NULL);

PyInterpreterState *interp = NULL;
int64_t interpid = -1;
PyObject *copied = NULL;
if (m_dict != NULL) {
interp = _PyInterpreterState_GET();
assert(value->origin != _Py_ext_module_origin_CORE);
assert(PyDict_Check(m_dict));
copied = PyDict_Copy(m_dict);
Expand All @@ -1001,6 +1002,11 @@ set_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict)
return -1;
}
// XXX We may want to make copied immortal.

value->def->m_base.m_cache_has_m_dict = 1;

PyInterpreterState *interp = _PyInterpreterState_GET();
interpid = PyInterpreterState_GetID(interp);
}

/* XXX gh-88216: The copied dict is owned by the current
Expand All @@ -1025,7 +1031,7 @@ set_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict)

value->def->m_base.m_copy = Py_XNewRef(copied);
value->m_dict = (cached_m_dict_t)copied;
value->m_dict_interp = interp;
value->m_dict_interpid = interpid;
return 0;
}

Expand Down Expand Up @@ -1079,7 +1085,7 @@ update_extensions_cache_value(struct extensions_cache_value *value,
/* For now we don't worry about comparing value->m_copy. */
assert(def->m_base.m_copy == NULL || m_dict != NULL);
assert(value->m_dict == NULL || m_dict != NULL);
assert((value->m_dict == NULL) == (value->m_dict_interp == NULL));
assert((value->m_dict == NULL) == (value->m_dict_interpid < 0));
assert(origin != _Py_ext_module_origin_CORE || m_dict == NULL);

/* We assume that all module defs are statically allocated
Expand Down Expand Up @@ -1261,7 +1267,7 @@ _extensions_cache_set(PyObject *path, PyObject *name,
struct extensions_cache_value *value = NULL;
struct extensions_cache_value *newvalue = NULL;

struct extensions_cache_value updates = {0};
struct extensions_cache_value updates = EXTENSIONS_CACHE_VALUE_INIT;
if (update_extensions_cache_value(
&updates, def, m_init, m_dict, origin) < 0)
{
Expand Down