bpo-46072: Add some object layout and allocation stats (GH-31051)

Author: markshannon

Include/internal/pycore_code.h

@@ -305,9 +305,19 @@ typedef struct _call_stats {
     uint64_t pyeval_calls;
 } CallStats;
 
+typedef struct _object_stats {
+    uint64_t allocations;
+    uint64_t frees;
+    uint64_t new_values;
+    uint64_t dict_materialized_on_request;
+    uint64_t dict_materialized_new_key;
+    uint64_t dict_materialized_too_big;
+} ObjectStats;
+
 typedef struct _stats {
     OpcodeStats opcode_stats[256];
     CallStats call_stats;
+    ObjectStats object_stats;
 } PyStats;
 
 extern PyStats _py_stats;
@@ -316,6 +326,7 @@ extern PyStats _py_stats;
 #define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
 #define OPCODE_EXE_INC(opname) _py_stats.opcode_stats[opname].execution_count++
 #define CALL_STAT_INC(name) _py_stats.call_stats.name++
+#define OBJECT_STAT_INC(name) _py_stats.object_stats.name++
 
 void _Py_PrintSpecializationStats(int to_file);
 
@@ -326,6 +337,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
 #define STAT_DEC(opname, name) ((void)0)
 #define OPCODE_EXE_INC(opname) ((void)0)
 #define CALL_STAT_INC(name) ((void)0)
+#define OBJECT_STAT_INC(name) ((void)0)
 #endif
 
 

Objects/dictobject.c

@@ -114,6 +114,7 @@ As a consequence of this, split keys have a maximum size of 16.
 #include "Python.h"
 #include "pycore_bitutils.h"      // _Py_bit_length
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
+#include "pycore_code.h"          // stats
 #include "pycore_dict.h"          // PyDictKeysObject
 #include "pycore_gc.h"            // _PyObject_GC_IS_TRACKED()
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
@@ -4990,6 +4991,7 @@ _PyObject_InitializeDict(PyObject *obj)
         return 0;
     }
     if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        OBJECT_STAT_INC(new_values);
         return init_inline_values(obj, tp);
     }
     PyObject *dict;
@@ -5033,6 +5035,7 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
 {
     assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
+    OBJECT_STAT_INC(dict_materialized_on_request);
     return make_dict_from_instance_attributes(keys, values);
 }
 
@@ -5051,6 +5054,14 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
             PyErr_SetObject(PyExc_AttributeError, name);
             return -1;
         }
+#ifdef Py_STATS
+        if (shared_keys_usable_size(keys) > 14) {
+            OBJECT_STAT_INC(dict_materialized_too_big);
+        }
+        else {
+            OBJECT_STAT_INC(dict_materialized_new_key);
+        }
+#endif
         PyObject *dict = make_dict_from_instance_attributes(keys, values);
         if (dict == NULL) {
             return -1;
@@ -5183,6 +5194,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
         PyObject **dictptr = _PyObject_ManagedDictPointer(obj);
         if (*values_ptr) {
             assert(*dictptr == NULL);
+            OBJECT_STAT_INC(dict_materialized_on_request);
             *dictptr = dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), *values_ptr);
             if (dict != NULL) {
                 *values_ptr = NULL;

Objects/obmalloc.c

@@ -1,5 +1,6 @@
 #include "Python.h"
 #include "pycore_pymem.h"         // _PyTraceMalloc_Config
+#include "pycore_code.h"         // stats
 
 #include <stdbool.h>
 #include <stdlib.h>               // malloc()
@@ -695,6 +696,7 @@ PyObject_Malloc(size_t size)
     /* see PyMem_RawMalloc() */
     if (size > (size_t)PY_SSIZE_T_MAX)
         return NULL;
+    OBJECT_STAT_INC(allocations);
     return _PyObject.malloc(_PyObject.ctx, size);
 }
 
@@ -704,6 +706,7 @@ PyObject_Calloc(size_t nelem, size_t elsize)
     /* see PyMem_RawMalloc() */
     if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
         return NULL;
+    OBJECT_STAT_INC(allocations);
     return _PyObject.calloc(_PyObject.ctx, nelem, elsize);
 }
 
@@ -719,6 +722,7 @@ PyObject_Realloc(void *ptr, size_t new_size)
 void
 PyObject_Free(void *ptr)
 {
+    OBJECT_STAT_INC(frees);
     _PyObject.free(_PyObject.ctx, ptr);
 }
 

Python/specialize.c

@@ -171,10 +171,22 @@ print_call_stats(FILE *out, CallStats *stats)
     fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
 }
 
+static void
+print_object_stats(FILE *out, ObjectStats *stats)
+{
+    fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);
+    fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
+    fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+    fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
+    fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
+    fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
+}
+
 static void
 print_stats(FILE *out, PyStats *stats) {
     print_spec_stats(out, stats->opcode_stats);
     print_call_stats(out, &stats->call_stats);
+    print_object_stats(out, &stats->object_stats);
 }
 
 void

Tools/scripts/summarize_stats.py

@@ -105,7 +105,17 @@ def main():
             total += value
     for key, value in stats.items():
         if "Calls to" in key:
-            print(f"{key}: {value} {100*value/total:0.1f}%")
+            print(f"    {key}: {value} {100*value/total:0.1f}%")
+    print("Object stats:")
+    total = stats.get("Object new values")
+    for key, value in stats.items():
+        if key.startswith("Object"):
+            if "materialize" in key:
+                print(f"    {key}: {value} {100*value/total:0.1f}%")
+            else:
+                print(f"    {key}: {value}")
+    total = 0
+
 
 if __name__ == "__main__":
     main()