[ty] Teach ty the meaning of desperation (try ancestor pyproject.tomls as search-paths if module resolution fails) (#21745)

## Summary

This makes an importing file a required argument to module resolution,
and if the fast-path cached query fails to resolve the module, take the
slow-path uncached (could be cached if we want)
`desperately_resolve_module` which will walk up from the importing file
until it finds a `pyproject.toml` (arbitrary decision, we could try
every ancestor directory), at which point it takes one last desperate
attempt to use that directory as a search-path. We do not continue
walking up once we've found a `pyproject.toml` (arbitrary decision, we
could keep going up).

Running locally, this fixes every broken-for-workspace-reasons import in
pyx's workspace!

* Fixes astral-sh/ty#1539
* Improves astral-sh/ty#839

## Test Plan

The workspace tests see a huge improvement on most absolute imports.

Author: Gankra

crates/ruff_graph/src/lib.rs

@@ -49,7 +49,7 @@ impl ModuleImports {
         // Resolve the imports.
         let mut resolved_imports = ModuleImports::default();
         for import in imports {
-            for resolved in Resolver::new(db).resolve(import) {
+            for resolved in Resolver::new(db, path).resolve(import) {
                 if let Some(path) = resolved.as_system_path() {
                     resolved_imports.insert(path.to_path_buf());
                 }

crates/ruff_graph/src/resolver.rs

@@ -1,18 +1,25 @@
-use ruff_db::files::FilePath;
-use ty_python_semantic::{ModuleName, resolve_module, resolve_real_module};
+use ruff_db::files::{File, FilePath, system_path_to_file};
+use ruff_db::system::SystemPath;
+use ty_python_semantic::{
+    ModuleName, resolve_module, resolve_module_confident, resolve_real_module,
+    resolve_real_module_confident,
+};
 
 use crate::ModuleDb;
 use crate::collector::CollectedImport;
 
 /// Collect all imports for a given Python file.
 pub(crate) struct Resolver<'a> {
     db: &'a ModuleDb,
+    file: Option<File>,
 }
 
 impl<'a> Resolver<'a> {
     /// Initialize a [`Resolver`] with a given [`ModuleDb`].
-    pub(crate) fn new(db: &'a ModuleDb) -> Self {
-        Self { db }
+    pub(crate) fn new(db: &'a ModuleDb, path: &SystemPath) -> Self {
+        // If we know the importing file we can potentially resolve more imports
+        let file = system_path_to_file(db, path).ok();
+        Self { db, file }
     }
 
     /// Resolve the [`CollectedImport`] into a [`FilePath`].
@@ -70,13 +77,21 @@ impl<'a> Resolver<'a> {
 
     /// Resolves a module name to a module.
     pub(crate) fn resolve_module(&self, module_name: &ModuleName) -> Option<&'a FilePath> {
-        let module = resolve_module(self.db, module_name)?;
+        let module = if let Some(file) = self.file {
+            resolve_module(self.db, file, module_name)?
+        } else {
+            resolve_module_confident(self.db, module_name)?
+        };
         Some(module.file(self.db)?.path(self.db))
     }
 
     /// Resolves a module name to a module (stubs not allowed).
     fn resolve_real_module(&self, module_name: &ModuleName) -> Option<&'a FilePath> {
-        let module = resolve_real_module(self.db, module_name)?;
+        let module = if let Some(file) = self.file {
+            resolve_real_module(self.db, file, module_name)?
+        } else {
+            resolve_real_module_confident(self.db, module_name)?
+        };
         Some(module.file(self.db)?.path(self.db))
     }
 }

crates/ty/tests/file_watching.rs

@@ -15,7 +15,7 @@ use ty_project::metadata::pyproject::{PyProject, Tool};
 use ty_project::metadata::value::{RangedValue, RelativePathBuf};
 use ty_project::watch::{ChangeEvent, ProjectWatcher, directory_watcher};
 use ty_project::{Db, ProjectDatabase, ProjectMetadata};
-use ty_python_semantic::{Module, ModuleName, PythonPlatform, resolve_module};
+use ty_python_semantic::{Module, ModuleName, PythonPlatform, resolve_module_confident};
 
 struct TestCase {
     db: ProjectDatabase,
@@ -232,7 +232,8 @@ impl TestCase {
     }
 
     fn module<'c>(&'c self, name: &str) -> Module<'c> {
-        resolve_module(self.db(), &ModuleName::new(name).unwrap()).expect("module to be present")
+        resolve_module_confident(self.db(), &ModuleName::new(name).unwrap())
+            .expect("module to be present")
     }
 
     fn sorted_submodule_names(&self, parent_module_name: &str) -> Vec<String> {
@@ -811,7 +812,8 @@ fn directory_moved_to_project() -> anyhow::Result<()> {
         .with_context(|| "Failed to create __init__.py")?;
     std::fs::write(a_original_path.as_std_path(), "").with_context(|| "Failed to create a.py")?;
 
-    let sub_a_module = resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap());
+    let sub_a_module =
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap());
 
     assert_eq!(sub_a_module, None);
     case.assert_indexed_project_files([bar]);
@@ -832,7 +834,9 @@ fn directory_moved_to_project() -> anyhow::Result<()> {
         .expect("a.py to exist");
 
     // `import sub.a` should now resolve
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()
+    );
 
     case.assert_indexed_project_files([bar, init_file, a_file]);
 
@@ -848,7 +852,9 @@ fn directory_moved_to_trash() -> anyhow::Result<()> {
     ])?;
     let bar = case.system_file(case.project_path("bar.py")).unwrap();
 
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()
+    );
 
     let sub_path = case.project_path("sub");
     let init_file = case
@@ -870,7 +876,9 @@ fn directory_moved_to_trash() -> anyhow::Result<()> {
     case.apply_changes(changes, None);
 
     // `import sub.a` should no longer resolve
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none()
+    );
 
     assert!(!init_file.exists(case.db()));
     assert!(!a_file.exists(case.db()));
@@ -890,8 +898,12 @@ fn directory_renamed() -> anyhow::Result<()> {
 
     let bar = case.system_file(case.project_path("bar.py")).unwrap();
 
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some());
-    assert!(resolve_module(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_none());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()
+    );
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_none()
+    );
 
     let sub_path = case.project_path("sub");
     let sub_init = case
@@ -915,9 +927,13 @@ fn directory_renamed() -> anyhow::Result<()> {
     case.apply_changes(changes, None);
 
     // `import sub.a` should no longer resolve
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none()
+    );
     // `import foo.baz` should now resolve
-    assert!(resolve_module(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_some());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("foo.baz").unwrap()).is_some()
+    );
 
     // The old paths are no longer tracked
     assert!(!sub_init.exists(case.db()));
@@ -950,7 +966,9 @@ fn directory_deleted() -> anyhow::Result<()> {
 
     let bar = case.system_file(case.project_path("bar.py")).unwrap();
 
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_some()
+    );
 
     let sub_path = case.project_path("sub");
 
@@ -970,7 +988,9 @@ fn directory_deleted() -> anyhow::Result<()> {
     case.apply_changes(changes, None);
 
     // `import sub.a` should no longer resolve
-    assert!(resolve_module(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none());
+    assert!(
+        resolve_module_confident(case.db(), &ModuleName::new_static("sub.a").unwrap()).is_none()
+    );
 
     assert!(!init_file.exists(case.db()));
     assert!(!a_file.exists(case.db()));
@@ -999,7 +1019,7 @@ fn search_path() -> anyhow::Result<()> {
     let site_packages = case.root_path().join("site_packages");
 
     assert_eq!(
-        resolve_module(case.db(), &ModuleName::new("a").unwrap()),
+        resolve_module_confident(case.db(), &ModuleName::new("a").unwrap()),
         None
     );
 
@@ -1009,7 +1029,7 @@ fn search_path() -> anyhow::Result<()> {
 
     case.apply_changes(changes, None);
 
-    assert!(resolve_module(case.db(), &ModuleName::new_static("a").unwrap()).is_some());
+    assert!(resolve_module_confident(case.db(), &ModuleName::new_static("a").unwrap()).is_some());
     case.assert_indexed_project_files([case.system_file(case.project_path("bar.py")).unwrap()]);
 
     Ok(())
@@ -1022,7 +1042,7 @@ fn add_search_path() -> anyhow::Result<()> {
     let site_packages = case.project_path("site_packages");
     std::fs::create_dir_all(site_packages.as_std_path())?;
 
-    assert!(resolve_module(case.db(), &ModuleName::new_static("a").unwrap()).is_none());
+    assert!(resolve_module_confident(case.db(), &ModuleName::new_static("a").unwrap()).is_none());
 
     // Register site-packages as a search path.
     case.update_options(Options {
@@ -1040,7 +1060,7 @@ fn add_search_path() -> anyhow::Result<()> {
 
     case.apply_changes(changes, None);
 
-    assert!(resolve_module(case.db(), &ModuleName::new_static("a").unwrap()).is_some());
+    assert!(resolve_module_confident(case.db(), &ModuleName::new_static("a").unwrap()).is_some());
 
     Ok(())
 }
@@ -1172,7 +1192,7 @@ fn changed_versions_file() -> anyhow::Result<()> {
 
     // Unset the custom typeshed directory.
     assert_eq!(
-        resolve_module(case.db(), &ModuleName::new("os").unwrap()),
+        resolve_module_confident(case.db(), &ModuleName::new("os").unwrap()),
         None
     );
 
@@ -1187,7 +1207,7 @@ fn changed_versions_file() -> anyhow::Result<()> {
 
     case.apply_changes(changes, None);
 
-    assert!(resolve_module(case.db(), &ModuleName::new("os").unwrap()).is_some());
+    assert!(resolve_module_confident(case.db(), &ModuleName::new("os").unwrap()).is_some());
 
     Ok(())
 }
@@ -1410,7 +1430,7 @@ mod unix {
             Ok(())
         })?;
 
-        let baz = resolve_module(case.db(), &ModuleName::new_static("bar.baz").unwrap())
+        let baz = resolve_module_confident(case.db(), &ModuleName::new_static("bar.baz").unwrap())
             .expect("Expected bar.baz to exist in site-packages.");
         let baz_project = case.project_path("bar/baz.py");
         let baz_file = baz.file(case.db()).unwrap();
@@ -1486,7 +1506,7 @@ mod unix {
             Ok(())
         })?;
 
-        let baz = resolve_module(case.db(), &ModuleName::new_static("bar.baz").unwrap())
+        let baz = resolve_module_confident(case.db(), &ModuleName::new_static("bar.baz").unwrap())
             .expect("Expected bar.baz to exist in site-packages.");
         let baz_file = baz.file(case.db()).unwrap();
         let bar_baz = case.project_path("bar/baz.py");
@@ -1591,7 +1611,7 @@ mod unix {
             Ok(())
         })?;
 
-        let baz = resolve_module(case.db(), &ModuleName::new_static("bar.baz").unwrap())
+        let baz = resolve_module_confident(case.db(), &ModuleName::new_static("bar.baz").unwrap())
             .expect("Expected bar.baz to exist in site-packages.");
         let baz_site_packages_path =
             case.project_path(".venv/lib/python3.12/site-packages/bar/baz.py");
@@ -1854,11 +1874,11 @@ fn rename_files_casing_only() -> anyhow::Result<()> {
     let mut case = setup([("lib.py", "class Foo: ...")])?;
 
     assert!(
-        resolve_module(case.db(), &ModuleName::new("lib").unwrap()).is_some(),
+        resolve_module_confident(case.db(), &ModuleName::new("lib").unwrap()).is_some(),
         "Expected `lib` module to exist."
     );
     assert_eq!(
-        resolve_module(case.db(), &ModuleName::new("Lib").unwrap()),
+        resolve_module_confident(case.db(), &ModuleName::new("Lib").unwrap()),
         None,
         "Expected `Lib` module not to exist"
     );
@@ -1891,13 +1911,13 @@ fn rename_files_casing_only() -> anyhow::Result<()> {
 
     // Resolving `lib` should now fail but `Lib` should now succeed
     assert_eq!(
-        resolve_module(case.db(), &ModuleName::new("lib").unwrap()),
+        resolve_module_confident(case.db(), &ModuleName::new("lib").unwrap()),
         None,
         "Expected `lib` module to no longer exist."
     );
 
     assert!(
-        resolve_module(case.db(), &ModuleName::new("Lib").unwrap()).is_some(),
+        resolve_module_confident(case.db(), &ModuleName::new("Lib").unwrap()).is_some(),
         "Expected `Lib` module to exist"
     );
 

crates/ty_ide/src/all_symbols.rs

@@ -20,7 +20,7 @@ pub fn all_symbols<'db>(
 
     let typing_extensions = ModuleName::new("typing_extensions").unwrap();
     let is_typing_extensions_available = importing_from.is_stub(db)
-        || resolve_real_shadowable_module(db, &typing_extensions).is_some();
+        || resolve_real_shadowable_module(db, importing_from, &typing_extensions).is_some();
 
     let results = std::sync::Mutex::new(Vec::new());
     {