Fix #3011: Handle overlapping submodule promotions gracefully in metaflow_extensions

examples/overlapping_submodule_promotions/01_single_extension_no_conflict.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Example 1 – Single Extension, No Conflict
+==========================================
+
+This is the baseline scenario: a single extension package promotes a
+submodule and everything works without warnings.
+
+What you should see when you run this script:
+  ✅ No warnings emitted.
+  ✅ The alias is registered correctly.
+"""
+
+import sys
+import os
+import types
+import warnings
+
+# Ensure the repo root is on the path so we can import metaflow
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from metaflow.extension_support import (
+    EXT_PKG,
+    alias_submodules,
+    _promoted_aliases,
+)
+
+
+def main():
+    # Clean slate
+    _promoted_aliases.clear()
+
+    # Simulate a single extension: metaflow_extensions.acme_corp.plugins
+    # that promotes "datatools" so it becomes available as
+    #   metaflow.plugins.datatools → metaflow_extensions.acme_corp.plugins.datatools
+    mod = types.ModuleType("%s.acme_corp.plugins" % EXT_PKG)
+    mod.__package__ = mod.__name__
+    mod.__mf_promote_submodules__ = ["datatools"]
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        aliases = alias_submodules(mod, "acme_corp", "plugins")
+
+    print("=== Example 1: Single Extension, No Conflict ===\n")
+    print("Aliases created:", list(aliases.keys()))
+    print("Warnings emitted:", len(caught))
+
+    if len(caught) == 0:
+        print("\n✅ No conflict – everything is clean.")
+    else:
+        print("\n❌ Unexpected warnings!")
+        for w in caught:
+            print("  ", w.message)
+
+
+if __name__ == "__main__":
+    main()

examples/overlapping_submodule_promotions/02_two_extensions_overlap_warning.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+Example 2 – Two Extensions Promote the Same Alias (Overlap Warning)
+====================================================================
+
+Two different extension packages both promote ``datatools`` under
+``metaflow.plugins``.  The extension system detects the conflict,
+emits a ``UserWarning``, and lets the **last-loaded** package win.
+
+What you should see when you run this script:
+  ⚠️  One warning identifying the conflicting packages.
+  ✅ The second package's target is the resolved alias.
+"""
+
+import sys
+import os
+import types
+import warnings
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from metaflow.extension_support import (
+    EXT_PKG,
+    alias_submodules,
+    _promoted_aliases,
+)
+
+
+def _make_module(org_name):
+    """Create a fake extension config module for *org_name*."""
+    mod = types.ModuleType("%s.%s.plugins" % (EXT_PKG, org_name))
+    mod.__package__ = mod.__name__
+    mod.__mf_promote_submodules__ = ["datatools"]
+    return mod
+
+
+def main():
+    _promoted_aliases.clear()
+
+    mod_alpha = _make_module("alpha_corp")
+    mod_beta = _make_module("beta_corp")
+
+    all_warnings = []
+
+    # --- First extension loads (alpha_corp) ---
+    with warnings.catch_warnings(record=True) as w1:
+        warnings.simplefilter("always")
+        alias_submodules(mod_alpha, "alpha_corp", "plugins")
+    all_warnings.extend(w1)
+
+    # --- Second extension loads (beta_corp) – overlap! ---
+    with warnings.catch_warnings(record=True) as w2:
+        warnings.simplefilter("always")
+        aliases = alias_submodules(mod_beta, "beta_corp", "plugins")
+    all_warnings.extend(w2)
+
+    print("=== Example 2: Two Extensions Overlap ===\n")
+    print("Final alias target:", aliases.get("metaflow.plugins.datatools"))
+    print("Warnings emitted:", len(all_warnings))
+
+    for w in all_warnings:
+        print("\n⚠️  WARNING:", w.message)
+
+    winner = _promoted_aliases.get("metaflow.plugins.datatools")
+    if winner:
+        print("\n✅ Winner (last-loaded):", winner[0], "→", winner[1])
+
+
+if __name__ == "__main__":
+    main()

examples/overlapping_submodule_promotions/03_partial_overlap.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""
+Example 3 – Partial Overlap
+============================
+
+Two extension packages promote several submodules each, but only **one**
+alias overlaps.  The system should warn about the overlapping alias only,
+leaving the unique ones untouched.
+
+What you should see when you run this script:
+  ⚠️  One warning (for ``common_utils`` only).
+  ✅ All other aliases are registered without warnings.
+"""
+
+import sys
+import os
+import types
+import warnings
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from metaflow.extension_support import (
+    EXT_PKG,
+    alias_submodules,
+    _promoted_aliases,
+)
+
+
+def main():
+    _promoted_aliases.clear()
+
+    # alpha_corp promotes: common_utils, alpha_special
+    mod_alpha = types.ModuleType("%s.alpha_corp.plugins" % EXT_PKG)
+    mod_alpha.__package__ = mod_alpha.__name__
+    mod_alpha.__mf_promote_submodules__ = ["common_utils", "alpha_special"]
+
+    # beta_corp promotes: common_utils, beta_special
+    mod_beta = types.ModuleType("%s.beta_corp.plugins" % EXT_PKG)
+    mod_beta.__package__ = mod_beta.__name__
+    mod_beta.__mf_promote_submodules__ = ["common_utils", "beta_special"]
+
+    all_warnings = []
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        aliases_a = alias_submodules(mod_alpha, "alpha_corp", "plugins")
+    all_warnings.extend(w)
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        aliases_b = alias_submodules(mod_beta, "beta_corp", "plugins")
+    all_warnings.extend(w)
+
+    print("=== Example 3: Partial Overlap ===\n")
+    print("Aliases from alpha_corp:", list(aliases_a.keys()))
+    print("Aliases from beta_corp:", list(aliases_b.keys()))
+    print("Total warnings:", len(all_warnings))
+
+    for w in all_warnings:
+        print("\n⚠️  WARNING:", w.message)
+
+    print("\nFinal promoted aliases:")
+    for alias, (pkg, target) in sorted(_promoted_aliases.items()):
+        print("  %s → %s (from %s)" % (alias, target, pkg))
+
+
+if __name__ == "__main__":
+    main()

examples/overlapping_submodule_promotions/04_three_way_overlap.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Example 4 – Three-Way Overlap
+==============================
+
+Three different extension packages all promote the same ``shared`` alias.
+The system emits **two** warnings (one each time a new package overrides
+the previous winner) and the very last package wins.
+
+What you should see when you run this script:
+  ⚠️  Two warnings.
+  ✅ The third package is the final winner.
+"""
+
+import sys
+import os
+import types
+import warnings
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from metaflow.extension_support import (
+    EXT_PKG,
+    alias_submodules,
+    _promoted_aliases,
+)
+
+
+def _make_module(org_name):
+    mod = types.ModuleType("%s.%s.plugins" % (EXT_PKG, org_name))
+    mod.__package__ = mod.__name__
+    mod.__mf_promote_submodules__ = ["shared"]
+    return mod
+
+
+def main():
+    _promoted_aliases.clear()
+
+    packages = ["alpha_corp", "beta_corp", "gamma_corp"]
+    all_warnings = []
+
+    for org in packages:
+        mod = _make_module(org)
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            alias_submodules(mod, org, "plugins")
+        all_warnings.extend(w)
+
+    print("=== Example 4: Three-Way Overlap ===\n")
+    print("Packages loaded (in order):", packages)
+    print("Total warnings:", len(all_warnings))
+
+    for i, w in enumerate(all_warnings, 1):
+        print("\n⚠️  Warning #%d: %s" % (i, w.message))
+
+    winner = _promoted_aliases.get("metaflow.plugins.shared")
+    if winner:
+        print("\n✅ Final winner: %s → %s" % (winner[0], winner[1]))
+
+
+if __name__ == "__main__":
+    main()

examples/overlapping_submodule_promotions/05_same_package_repromotion.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+Example 5 – Same Package Re-promotion (No Warning)
+====================================================
+
+If the *same* package promotes the same alias more than once (e.g. because
+the module is loaded twice), no warning should be raised – there is no
+actual conflict.
+
+What you should see when you run this script:
+  ✅ Zero warnings.
+"""
+
+import sys
+import os
+import types
+import warnings
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from metaflow.extension_support import (
+    EXT_PKG,
+    alias_submodules,
+    _promoted_aliases,
+)
+
+
+def main():
+    _promoted_aliases.clear()
+
+    mod = types.ModuleType("%s.acme_corp.plugins" % EXT_PKG)
+    mod.__package__ = mod.__name__
+    mod.__mf_promote_submodules__ = ["datatools"]
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        # Promote twice from the same package
+        alias_submodules(mod, "acme_corp", "plugins")
+        alias_submodules(mod, "acme_corp", "plugins")
+
+    print("=== Example 5: Same Package Re-promotion ===\n")
+    print("Warnings emitted:", len(caught))
+
+    if len(caught) == 0:
+        print("✅ No conflict – same package re-promoting is harmless.")
+    else:
+        print("❌ Unexpected warnings!")
+        for w in caught:
+            print("  ", w.message)
+
+
+if __name__ == "__main__":
+    main()

examples/overlapping_submodule_promotions/README.md

@@ -0,0 +1,64 @@
+# Overlapping Submodule Promotions in `metaflow_extensions`
+
+This directory contains examples that demonstrate how the Metaflow extension
+system now handles **overlapping submodule promotions** – the situation where
+two or more extension packages promote the same alias via
+`__mf_promote_submodules__`.
+
+## Background
+
+Metaflow extensions are Python packages under the `metaflow_extensions`
+namespace.  Each extension can *promote* submodules so they become accessible
+under a short `metaflow.*` alias (e.g. `metaflow.plugins.datatools` →
+`metaflow_extensions.my_org.plugins.datatools`).
+
+The promotion is declared by setting `__mf_promote_submodules__` in the
+extension's config module:
+
+```python
+# metaflow_extensions/my_org/plugins/mfextinit_my_org.py
+__mf_promote_submodules__ = ["datatools"]
+```
+
+### The Problem (Issue #3011)
+
+Before this fix, if **two** extension packages both promoted the same alias
+(e.g. both declared `["datatools"]`), the conflict was handled **silently** –
+the second package's target would simply overwrite the first one with no
+indication to the user.
+
+### The Fix
+
+The extension system now:
+
+1. **Detects** overlapping promotions from *different* packages.
+2. **Emits a `UserWarning`** that identifies both the overriding and overridden
+   packages, the alias in question, and the concrete target modules.
+3. **Resolves deterministically**: the **last-loaded** package wins. Load order
+   is determined by topological sort of package dependencies, then alphabetical
+   order for ties – exactly as it was before, but now *documented* and *visible*.
+4. If the **same** package re-promotes the same alias (e.g. loaded twice), no
+   warning is emitted because there is no actual conflict.
+
+## Examples in this Directory
+
+| File | Description |
+|------|-------------|
+| `01_single_extension_no_conflict.py` | Baseline: a single extension promoting submodules works cleanly. |
+| `02_two_extensions_overlap_warning.py` | Two extensions promote the same alias; demonstrates the warning. |
+| `03_partial_overlap.py` | Two extensions with only *some* overlapping aliases. |
+| `04_three_way_overlap.py` | Three extensions competing for the same alias; two warnings are emitted. |
+| `05_same_package_repromotion.py` | Same package promoting twice – no warning. |
+
+## How to Run
+
+Each example is a standalone Python script.  Run from the repository root:
+
+```bash
+python3 examples/overlapping_submodule_promotions/01_single_extension_no_conflict.py
+```
+
+> **Note:** These examples use the *internal* `alias_submodules` API with
+> synthetic (fake) modules.  They are designed to be educational – they show
+> exactly what happens inside the extension system without requiring you to
+> install actual extension packages.