C API for module creation (#59604)

Provide a way to create new modules without resorting to passing an
`Expr(:module, ...)` to `eval()` by splitting up `jl_eval_module_expr()`
into three pieces:

Two module creation functions which are exported:

* `jl_begin_new_module()`
  - Creates the module
- Situates it within the module hierarchy (creating a binding within the
parent module, setting the module parent and inheriting uuid,
registering root modules)
- Deals with global rooting during initialization and state for
detecting whether the module is open for eval.
- Creates standard imports and bindings for module-local eval and
include
* `jl_end_new_module()` cleans up some of this state and calls
`__init__()` for the module and its children when necessary.

The `Expr`-related machinery is left in the internal function
`jl_eval_module_expr()`.

Something along these lines is required to avoid calling
`eval(Expr(:module, ...))` in the JuliaLowering implementation of module
evaluation where we try never to construct `Expr`.

There's clearly further cleanup which could be done here. It would be
nice to use less global state (`jl_current_modules` and
`jl_module_init_order`) and to not to have the `Base.__toplevel__` hack.
However I wanted to keep this PR small so I've resisted the temptation
to change anything more for now.

Similar to #57965 in the sense that it splits more functionality out of
`eval()`.

Author: c42f

src/jl_exported_funcs.inc

@@ -330,6 +330,8 @@
     XX(jl_new_method_table) \
     XX(jl_new_method_uninit) \
     XX(jl_new_module) \
+    XX(jl_begin_new_module) \
+    XX(jl_end_new_module) \
     XX(jl_new_opaque_closure_from_code_info) \
     XX(jl_new_opaque_closure_from_code_info_in_world) \
     XX(jl_new_primitivetype) \

src/toplevel.c

@@ -30,6 +30,8 @@ JL_DLLEXPORT _Atomic(int) jl_lineno = 0; // need to update jl_fprint_critical_er
 // current file name
 JL_DLLEXPORT _Atomic(const char *) jl_filename = "none"; // need to update jl_fprint_critical_error if this is TLS
 
+static jl_value_t *jl_eval_toplevel_stmts(jl_module_t *JL_NONNULL m, jl_array_t *stmts, int fast, int need_value, const char **toplevel_filename, int *toplevel_lineno);
+
 htable_t jl_current_modules;
 jl_mutex_t jl_modules_mutex;
 
@@ -109,25 +111,10 @@ static int jl_is__toplevel__mod(jl_module_t *mod, jl_task_t *ct)
         (jl_value_t*)mod == jl_get_global_value(jl_base_module, jl_symbol("__toplevel__"), ct->world_age);
 }
 
-// TODO: add locks around global state mutation operations
-static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex)
+JL_DLLEXPORT jl_module_t *jl_begin_new_module(jl_module_t *parent_module, jl_sym_t *name,
+                                              int std_imports, const char *filename, int lineno)
 {
     jl_task_t *ct = jl_current_task;
-    assert(ex->head == jl_module_sym);
-    if (jl_array_nrows(ex->args) != 3 || !jl_is_expr(jl_exprarg(ex, 2))) {
-        jl_error("syntax: malformed module expression");
-    }
-
-    if (((jl_expr_t *)(jl_exprarg(ex, 2)))->head != jl_symbol("block")) {
-        jl_error("syntax: module expression third argument must be a block");
-    }
-
-    int std_imports = (jl_exprarg(ex, 0) == jl_true);
-    jl_sym_t *name = (jl_sym_t*)jl_exprarg(ex, 1);
-    if (!jl_is_symbol(name)) {
-        jl_type_error("module", (jl_value_t*)jl_symbol_type, (jl_value_t*)name);
-    }
-
     int is_parent__toplevel__ = jl_is__toplevel__mod(parent_module, ct);
     // If we have `Base`, don't also try to import `Core` - the `Base` exports are a superset.
     // While we allow multiple imports of the same binding from different modules, various error printing
@@ -140,18 +127,6 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
     JL_UNLOCK(&jl_modules_mutex);
     // copy parent environment info into submodule
     newm->uuid = parent_module->uuid;
-    jl_array_t *exprs = ((jl_expr_t*)jl_exprarg(ex, 2))->args;
-    int lineno = 0;
-    const char *filename = "none";
-    if (jl_array_nrows(exprs) > 0) {
-        jl_value_t *lineex = jl_array_ptr_ref(exprs, 0);
-        if (jl_is_linenode(lineex)) {
-            lineno = jl_linenode_line(lineex);
-            jl_value_t *file = jl_linenode_file(lineex);
-            if (jl_is_symbol(file))
-                filename = jl_symbol_name((jl_sym_t*)file);
-        }
-    }
     newm->file = jl_symbol(filename);
     jl_gc_wb_knownold(newm, newm->file);
     newm->line = lineno;
@@ -171,8 +146,6 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
         }
     }
 
-    size_t last_age = ct->world_age;
-
     if (parent_module == jl_main_module && name == jl_symbol("Base") && jl_base_module == NULL) {
         // pick up Base module during bootstrap
         jl_base_module = newm;
@@ -187,15 +160,14 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
     else {
         jl_declare_constant_val(NULL, parent_module, name, (jl_value_t*)newm);
     }
+    JL_GC_POP();
 
-    for (int i = 0; i < jl_array_nrows(exprs); i++) {
-        // process toplevel form
-        form = jl_svecref(jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0), 0);
-        ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-        (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno);
-    }
-    ct->world_age = last_age;
+    return newm;
+}
 
+JL_DLLEXPORT void jl_end_new_module(jl_module_t *newm) {
+    jl_value_t *form = NULL;
+    JL_GC_PUSH1(&form);
     JL_LOCK(&jl_modules_mutex);
     uintptr_t *refcnt = (uintptr_t*)ptrhash_bp(&jl_current_modules, (void*)newm);
     assert(*refcnt > (uintptr_t)HT_NOTFOUND);
@@ -206,10 +178,9 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
         jl_module_init_order = jl_alloc_vec_any(0);
     jl_array_ptr_1d_push(jl_module_init_order, (jl_value_t*)newm);
 
-    // defer init of children until parent is done being defined
-    // then initialize all in definition-finished order
-    // at build time, don't run them at all (defer for runtime)
-    form = NULL;
+    // Defer init of direct children until parent is done being defined then
+    // initialize all in definition-finished order.
+    // At build time, don't run them at all - defer for runtime
     if (!jl_generating_output()) {
         if (!ptrhash_has(&jl_current_modules, (void*)newm->parent)) {
             size_t i, l = jl_array_nrows(jl_module_init_order);
@@ -240,6 +211,43 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
     }
 
     JL_GC_POP();
+}
+
+static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex, const char **toplevel_filename, int *toplevel_lineno)
+{
+    assert(ex->head == jl_module_sym);
+    if (jl_array_nrows(ex->args) != 3 || !jl_is_expr(jl_exprarg(ex, 2))) {
+        jl_error("syntax: malformed module expression");
+    }
+
+    if (((jl_expr_t *)(jl_exprarg(ex, 2)))->head != jl_symbol("block")) {
+        jl_error("syntax: module expression third argument must be a block");
+    }
+    jl_array_t *stmts = ((jl_expr_t*)jl_exprarg(ex, 2))->args;
+
+    int std_imports = (jl_exprarg(ex, 0) == jl_true);
+    jl_sym_t *name = (jl_sym_t*)jl_exprarg(ex, 1);
+    if (!jl_is_symbol(name)) {
+        jl_type_error("module", (jl_value_t*)jl_symbol_type, (jl_value_t*)name);
+    }
+
+    int lineno = 0;
+    const char *filename = "none";
+    if (jl_array_nrows(stmts) > 0) {
+        jl_value_t *lineex = jl_array_ptr_ref(stmts, 0);
+        if (jl_is_linenode(lineex)) {
+            lineno = jl_linenode_line(lineex);
+            jl_value_t *file = jl_linenode_file(lineex);
+            if (jl_is_symbol(file))
+                filename = jl_symbol_name((jl_sym_t*)file);
+        }
+    }
+
+    jl_module_t *newm = jl_begin_new_module(parent_module, name, std_imports, filename, lineno);
+    JL_GC_PROMISE_ROOTED(newm); // Rooted in jl_current_modules
+    jl_eval_toplevel_stmts(newm, stmts, 1, 0, toplevel_filename, toplevel_lineno);
+    jl_end_new_module(newm);
+
     return (jl_value_t*)newm;
 }
 
@@ -596,6 +604,28 @@ JL_DLLEXPORT void jl_eval_const_decl(jl_module_t *m, jl_value_t *arg, jl_value_t
     jl_declare_constant_val(b, gm, gs, val);
 }
 
+static jl_value_t *jl_eval_toplevel_stmts(jl_module_t *JL_NONNULL m, jl_array_t *stmts, int fast, int need_value, const char **toplevel_filename, int *toplevel_lineno)
+{
+    jl_task_t *ct = jl_current_task;
+    size_t last_age = ct->world_age;
+    jl_value_t *root = NULL;
+    JL_GC_PUSH1(&root);
+    jl_value_t *res = jl_nothing;
+    int i;
+    for (i = 0; i < jl_array_nrows(stmts); i++) {
+        root = jl_array_ptr_ref(stmts, i);
+        if (jl_needs_lowering(root)) {
+            root = jl_svecref(jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0,
+                                       need_value), 0);
+        }
+        ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
+        res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno);
+    }
+    ct->world_age = last_age;
+    JL_GC_POP();
+    return res;
+}
+
 JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno)
 {
     jl_task_t *ct = jl_current_task;
@@ -654,7 +684,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val
     jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL;
 
     if (head == jl_module_sym) {
-        jl_value_t *val = jl_eval_module_expr(m, ex);
+        jl_value_t *val = jl_eval_module_expr(m, ex, toplevel_filename, toplevel_lineno);
         JL_GC_POP();
         return val;
     }
@@ -701,17 +731,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val
         return jl_nothing;
     }
     else if (head == jl_toplevel_sym) {
-        jl_value_t *res = jl_nothing;
-        int i;
-        for (i = 0; i < jl_array_nrows(ex->args); i++) {
-            root = jl_array_ptr_ref(ex->args, i);
-            if (jl_needs_lowering(root)) {
-                root = jl_svecref(jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0);
-            }
-            ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-            res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno);
-        }
-        ct->world_age = last_age;
+        jl_value_t *res = jl_eval_toplevel_stmts(m, ex->args, fast, 1,
+                                                 toplevel_filename, toplevel_lineno);
         JL_GC_POP();
         return res;
     }