bpo-45575: Use a less ad-hoc approach to freelists, and use them for int and float freelists.

Include/internal/pycore_floatobject.h

@@ -57,9 +57,6 @@ PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
 PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
 
 
-PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out);
-
-
 /* Format the object based on the format_spec, as defined in PEP 3101
    (Advanced String Formatting). */
 PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter(

Include/internal/pycore_interp.h

@@ -12,6 +12,7 @@ extern "C" {
 #include "pycore_ast_state.h"     // struct ast_state
 #include "pycore_gil.h"           // struct _gil_runtime_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
+#include "pycore_pymem.h"         // free lists
 #include "pycore_warnings.h"      // struct _warnings_runtime_state
 
 struct _pending_calls {
@@ -86,8 +87,8 @@ struct _Py_unicode_state {
 };
 
 #ifndef WITH_FREELISTS
+#  define WITH_FREELISTS 0
 // without freelists
-#  define PyFloat_MAXFREELIST 0
 // for tuples only store empty tuple singleton
 #  define PyTuple_MAXSAVESIZE 1
 #  define PyTuple_MAXFREELIST 1
@@ -98,20 +99,6 @@ struct _Py_unicode_state {
 #  define PyContext_MAXFREELIST 0
 #endif
 
-#ifndef PyFloat_MAXFREELIST
-#  define PyFloat_MAXFREELIST   100
-#endif
-
-struct _Py_float_state {
-#if PyFloat_MAXFREELIST > 0
-    /* Special free list
-       free_list is a singly-linked list of available PyFloatObjects,
-       linked via abuse of their ob_type members. */
-    int numfree;
-    PyFloatObject *free_list;
-#endif
-};
-
 /* Speed optimization to avoid frequent malloc/free of small tuples */
 #ifndef PyTuple_MAXSAVESIZE
    // Largest tuple to save on free list
@@ -271,6 +258,10 @@ struct _is {
 
     int finalizing;
 
+#if WITH_FREELISTS
+    _PyFreeList small_object_freelist;
+#endif
+
     struct _ceval_state ceval;
     struct _gc_runtime_state gc;
 
@@ -336,7 +327,6 @@ struct _is {
     PyLongObject* small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS];
     struct _Py_bytes_state bytes;
     struct _Py_unicode_state unicode;
-    struct _Py_float_state float_state;
     /* Using a cache is very effective since typically only a single slice is
        created and then deleted again. */
     PySliceObject *slice_cache;
@@ -348,7 +338,6 @@ struct _is {
     struct _Py_async_gen_state async_gen;
     struct _Py_context_state context;
     struct _Py_exc_state exc_state;
-
     struct ast_state ast;
     struct type_cache type_cache;
 };

Include/internal/pycore_pymem.h

@@ -100,6 +100,52 @@ void *_PyObject_VirtualAlloc(size_t size);
 void _PyObject_VirtualFree(void *, size_t size);
 
 
+#if WITH_FREELISTS
+/* Free lists.
+ *
+ * Free lists have a pointer to their first entry and
+ * the amount of space available allowing fast checks
+ * for emptiness and fullness.
+ * When empty they are half filled and when full they are
+ * completely emptied. This helps the underlying allocator
+ * avoid fragmentation and helps performance.
+ */
+
+typedef struct _freelist {
+    void *ptr;
+    uint32_t space;
+    uint16_t size;
+    uint16_t capacity;
+} _PyFreeList;
+
+extern void *_PyFreeList_HalfFillAndAllocate(_PyFreeList *list);
+extern void _PyFreeList_FreeToFull(_PyFreeList *list, void *ptr);
+
+static inline void *
+_PyFreeList_Alloc(_PyFreeList *list) {
+    if (list->ptr != NULL) {
+        void *result = list->ptr;
+        list->ptr = *((void **)result);
+        list->space++;
+        return result;
+    }
+    return _PyFreeList_HalfFillAndAllocate(list);
+}
+
+static inline void
+_PyFreeList_Free(_PyFreeList *list, void *ptr) {
+    if (list->space) {
+        *((void **)ptr) = list->ptr;
+        list->ptr = ptr;
+        list->space--;
+        return;
+    }
+    _PyFreeList_FreeToFull(list, ptr);
+}
+
+extern void _PyFreeList_Clear(_PyFreeList *list);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Misc/NEWS.d/next/Core and Builtins/2021-10-22-17-23-15.bpo-45575._oOrzd.rst

@@ -0,0 +1 @@
+Use a common free list for both ints and floats.

Objects/floatobject.c

@@ -10,6 +10,7 @@
 #include "pycore_long.h"          // _PyLong_GetOne()
 #include "pycore_object.h"        // _PyObject_Init()
 #include "pycore_pymath.h"        // _Py_ADJUST_ERANGE1()
+#include "pycore_pymem.h"           // Free lists
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
 #include <ctype.h>
@@ -23,21 +24,6 @@ class float "PyObject *" "&PyFloat_Type"
 
 #include "clinic/floatobject.c.h"
 
-#ifndef PyFloat_MAXFREELIST
-#  define PyFloat_MAXFREELIST   100
-#endif
-
-
-#if PyFloat_MAXFREELIST > 0
-static struct _Py_float_state *
-get_float_state(void)
-{
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    return &interp->float_state;
-}
-#endif
-
-
 double
 PyFloat_GetMax(void)
 {
@@ -128,25 +114,14 @@ PyFloat_GetInfo(void)
 PyObject *
 PyFloat_FromDouble(double fval)
 {
-    PyFloatObject *op;
-#if PyFloat_MAXFREELIST > 0
-    struct _Py_float_state *state = get_float_state();
-    op = state->free_list;
-    if (op != NULL) {
-#ifdef Py_DEBUG
-        // PyFloat_FromDouble() must not be called after _PyFloat_Fini()
-        assert(state->numfree != -1);
-#endif
-        state->free_list = (PyFloatObject *) Py_TYPE(op);
-        state->numfree--;
-    }
-    else
+#if WITH_FREELISTS
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    PyFloatObject *op = (PyFloatObject *)_PyFreeList_Alloc(&interp->small_object_freelist);
+#else
+    PyFloatObject *op = (PyFloatObject *)PyObject_Malloc(sizeof(PyFloatObject));
 #endif
-    {
-        op = PyObject_Malloc(sizeof(PyFloatObject));
-        if (!op) {
-            return PyErr_NoMemory();
-        }
+    if (op == NULL) {
+        return PyErr_NoMemory();
     }
     _PyObject_Init((PyObject*)op, &PyFloat_Type);
     op->ob_fval = fval;
@@ -239,20 +214,10 @@ PyFloat_FromString(PyObject *v)
 static void
 float_dealloc(PyFloatObject *op)
 {
-#if PyFloat_MAXFREELIST > 0
+#if WITH_FREELISTS
     if (PyFloat_CheckExact(op)) {
-        struct _Py_float_state *state = get_float_state();
-#ifdef Py_DEBUG
-        // float_dealloc() must not be called after _PyFloat_Fini()
-        assert(state->numfree != -1);
-#endif
-        if (state->numfree >= PyFloat_MAXFREELIST)  {
-            PyObject_Free(op);
-            return;
-        }
-        state->numfree++;
-        Py_SET_TYPE(op, (PyTypeObject *)state->free_list);
-        state->free_list = op;
+        PyInterpreterState *interp = _PyInterpreterState_GET();
+        _PyFreeList_Free(&interp->small_object_freelist, op);
     }
     else
 #endif
@@ -2045,39 +2010,15 @@ _PyFloat_InitTypes(void)
 void
 _PyFloat_ClearFreeList(PyInterpreterState *interp)
 {
-#if PyFloat_MAXFREELIST > 0
-    struct _Py_float_state *state = &interp->float_state;
-    PyFloatObject *f = state->free_list;
-    while (f != NULL) {
-        PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
-        PyObject_Free(f);
-        f = next;
-    }
-    state->free_list = NULL;
-    state->numfree = 0;
+#if WITH_FREELISTS
+    _PyFreeList_Clear(&interp->small_object_freelist);
 #endif
 }
 
 void
 _PyFloat_Fini(PyInterpreterState *interp)
 {
     _PyFloat_ClearFreeList(interp);
-#if defined(Py_DEBUG) && PyFloat_MAXFREELIST > 0
-    struct _Py_float_state *state = &interp->float_state;
-    state->numfree = -1;
-#endif
-}
-
-/* Print summary info about the state of the optimized allocator */
-void
-_PyFloat_DebugMallocStats(FILE *out)
-{
-#if PyFloat_MAXFREELIST > 0
-    struct _Py_float_state *state = get_float_state();
-    _PyDebugAllocatorStats(out,
-                           "free PyFloatObject",
-                           state->numfree, sizeof(PyFloatObject));
-#endif
 }
 
 

Objects/longobject.c

@@ -7,6 +7,7 @@
 #include "pycore_interp.h"        // _PY_NSMALLPOSINTS
 #include "pycore_long.h"          // __PyLong_GetSmallInt_internal()
 #include "pycore_object.h"        // _PyObject_InitVar()
+#include "pycore_pymem.h"           // Free lists
 #include "pycore_pystate.h"       // _Py_IsMainInterpreter()
 
 #include <ctype.h>
@@ -112,6 +113,15 @@ long_normalize(PyLongObject *v)
 #define MAX_LONG_DIGITS \
     ((PY_SSIZE_T_MAX - offsetof(PyLongObject, ob_digit))/sizeof(digit))
 
+union _small_object {
+    PyFloatObject f;
+    PyLongObject l;
+};
+
+/* Free list is shared between ints and floats as they take the same amount of memory (on most platforms)
+ */
+#define MAX_DIGITS_FOR_FREELIST ((sizeof(union _small_object) - offsetof(PyLongObject, ob_digit))/sizeof(digit))
+
 PyLongObject *
 _PyLong_New(Py_ssize_t size)
 {
@@ -121,16 +131,24 @@ _PyLong_New(Py_ssize_t size)
                         "too many digits in integer");
         return NULL;
     }
-    /* Fast operations for single digit integers (including zero)
-     * assume that there is always at least one digit present. */
-    Py_ssize_t ndigits = size ? size : 1;
-    /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) +
-       sizeof(digit)*size.  Previous incarnations of this code used
-       sizeof(PyVarObject) instead of the offsetof, but this risks being
-       incorrect in the presence of padding between the PyVarObject header
-       and the digits. */
-    result = PyObject_Malloc(offsetof(PyLongObject, ob_digit) +
-                             ndigits*sizeof(digit));
+#if WITH_FREELISTS
+    if (((size_t)size) <= MAX_DIGITS_FOR_FREELIST) {
+        PyInterpreterState *interp = _PyInterpreterState_GET();
+        result = (PyLongObject *)_PyFreeList_Alloc(&interp->small_object_freelist);
+#else
+    if (size == 0) {
+        result = PyObject_Malloc(sizeof(PyLongObject));
+#endif
+    }
+    else {
+        /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) +
+        sizeof(digit)*size.  Previous incarnations of this code used
+        sizeof(PyVarObject) instead of the offsetof, but this risks being
+        incorrect in the presence of padding between the PyVarObject header
+        and the digits. */
+        result = PyObject_Malloc(offsetof(PyLongObject, ob_digit) +
+                                size*sizeof(digit));
+    }
     if (!result) {
         PyErr_NoMemory();
         return NULL;
@@ -170,11 +188,14 @@ _PyLong_FromMedium(sdigit x)
 {
     assert(!IS_SMALL_INT(x));
     assert(is_medium_int(x));
-    /* We could use a freelist here */
-    PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
+#if WITH_FREELISTS
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    PyLongObject *v = (PyLongObject *)_PyFreeList_Alloc(&interp->small_object_freelist);
+#else
+    PyLongObject *v = (PyLongObject *)PyObject_Malloc(sizeof(PyLongObject));
+#endif
     if (v == NULL) {
-        PyErr_NoMemory();
-        return NULL;
+        return PyErr_NoMemory();
     }
     Py_ssize_t sign = x < 0 ? -1: 1;
     digit abs_x = x < 0 ? -x : x;
@@ -5743,12 +5764,28 @@ static PyNumberMethods long_as_number = {
     long_long,                  /* nb_index */
 };
 
+static void
+int_dealloc(PyLongObject *op)
+{
+#if WITH_FREELISTS
+    if (PyLong_CheckExact(op) && IS_MEDIUM_VALUE(op)) {
+        PyInterpreterState *interp = _PyInterpreterState_GET();
+        _PyFreeList_Free(&interp->small_object_freelist, op);
+    }
+    else
+#endif
+    {
+        Py_TYPE(op)->tp_free((PyObject *)op);
+    }
+}
+
+
 PyTypeObject PyLong_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "int",                                      /* tp_name */
     offsetof(PyLongObject, ob_digit),           /* tp_basicsize */
     sizeof(digit),                              /* tp_itemsize */
-    0,                                          /* tp_dealloc */
+    (destructor)int_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_vectorcall_offset */
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */

Objects/object.c

@@ -2079,7 +2079,6 @@ void
 _PyObject_DebugTypeStats(FILE *out)
 {
     _PyDict_DebugMallocStats(out);
-    _PyFloat_DebugMallocStats(out);
     _PyFrame_DebugMallocStats(out);
     _PyList_DebugMallocStats(out);
     _PyTuple_DebugMallocStats(out);