[ty] Fix some configuration panics in the LSP (#22040)

## Summary

This is a revival of #21047 now
that we have a reproducer again.

* Fixes astral-sh/ty#2031
* Fixes astral-sh/ty#859

## Test Plan

e2e test from @zanieb

---------

Co-authored-by: Zanie Blue <contact@zanie.dev>

Author: Gankra

crates/ruff_db/src/system/test.rs

@@ -1,5 +1,6 @@
 use glob::PatternError;
 use ruff_notebook::{Notebook, NotebookError};
+use rustc_hash::FxHashMap;
 use std::panic::RefUnwindSafe;
 use std::sync::{Arc, Mutex};
 
@@ -20,18 +21,44 @@ use super::walk_directory::WalkDirectoryBuilder;
 ///
 /// ## Warning
 /// Don't use this system for production code. It's intended for testing only.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct TestSystem {
     inner: Arc<dyn WritableSystem + RefUnwindSafe + Send + Sync>,
+    /// Environment variable overrides. If a key is present here, it takes precedence
+    /// over the inner system's environment variables.
+    env_overrides: Arc<Mutex<FxHashMap<String, Option<String>>>>,
+}
+
+impl Clone for TestSystem {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            env_overrides: self.env_overrides.clone(),
+        }
+    }
 }
 
 impl TestSystem {
     pub fn new(inner: impl WritableSystem + RefUnwindSafe + Send + Sync + 'static) -> Self {
         Self {
             inner: Arc::new(inner),
+            env_overrides: Arc::new(Mutex::new(FxHashMap::default())),
         }
     }
 
+    /// Sets an environment variable override. This takes precedence over the inner system.
+    pub fn set_env_var(&self, name: impl Into<String>, value: impl Into<String>) {
+        self.env_overrides
+            .lock()
+            .unwrap()
+            .insert(name.into(), Some(value.into()));
+    }
+
+    /// Removes an environment variable override, making it appear as not set.
+    pub fn remove_env_var(&self, name: impl Into<String>) {
+        self.env_overrides.lock().unwrap().insert(name.into(), None);
+    }
+
     /// Returns the [`InMemorySystem`].
     ///
     /// ## Panics
@@ -147,6 +174,18 @@ impl System for TestSystem {
         self.system().case_sensitivity()
     }
 
+    fn env_var(&self, name: &str) -> std::result::Result<String, std::env::VarError> {
+        // Check overrides first
+        if let Some(override_value) = self.env_overrides.lock().unwrap().get(name) {
+            return match override_value {
+                Some(value) => Ok(value.clone()),
+                None => Err(std::env::VarError::NotPresent),
+            };
+        }
+        // Fall back to inner system
+        self.system().env_var(name)
+    }
+
     fn dyn_clone(&self) -> Box<dyn System> {
         Box::new(self.clone())
     }
@@ -156,6 +195,7 @@ impl Default for TestSystem {
     fn default() -> Self {
         Self {
             inner: Arc::new(InMemorySystem::default()),
+            env_overrides: Arc::new(Mutex::new(FxHashMap::default())),
         }
     }
 }

crates/ty/tests/cli/python_environment.rs

@@ -2703,3 +2703,51 @@ fn pythonpath_multiple_dirs_is_respected() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+/// Test behavior when `VIRTUAL_ENV` is set but points to a non-existent path.
+#[test]
+fn missing_virtual_env() -> anyhow::Result<()> {
+    let working_venv_package1_path = if cfg!(windows) {
+        "project/.venv/Lib/site-packages/package1/__init__.py"
+    } else {
+        "project/.venv/lib/python3.13/site-packages/package1/__init__.py"
+    };
+
+    let case = CliTest::with_files([
+        (
+            "project/test.py",
+            r#"
+            from package1 import WorkingVenv
+            "#,
+        ),
+        (
+            "project/.venv/pyvenv.cfg",
+            r#"
+home = ./
+
+            "#,
+        ),
+        (
+            working_venv_package1_path,
+            r#"
+            class WorkingVenv: ...
+            "#,
+        ),
+    ])?;
+
+    assert_cmd_snapshot!(case.command()
+        .current_dir(case.root().join("project"))
+        .env("VIRTUAL_ENV", case.root().join("nonexistent-venv")), @r"
+    success: false
+    exit_code: 2
+    ----- stdout -----
+
+    ----- stderr -----
+    ty failed
+      Cause: Failed to discover local Python environment
+      Cause: Invalid `VIRTUAL_ENV` environment variable `<temp_dir>/nonexistent-venv`: does not point to a directory on disk
+      Cause: No such file or directory (os error 2)
+    ");
+
+    Ok(())
+}

crates/ty_project/src/metadata.rs

@@ -5,7 +5,7 @@ use ruff_python_ast::name::Name;
 use std::sync::Arc;
 use thiserror::Error;
 use ty_combine::Combine;
-use ty_python_semantic::ProgramSettings;
+use ty_python_semantic::{MisconfigurationMode, ProgramSettings};
 
 use crate::metadata::options::ProjectOptionsOverrides;
 use crate::metadata::pyproject::{Project, PyProject, PyProjectError, ResolveRequiresPythonError};
@@ -37,6 +37,9 @@ pub struct ProjectMetadata {
     /// The path ordering doesn't imply precedence.
     #[cfg_attr(test, serde(skip_serializing_if = "Vec::is_empty"))]
     pub(super) extra_configuration_paths: Vec<SystemPathBuf>,
+
+    #[cfg_attr(test, serde(skip))]
+    pub(super) misconfiguration_mode: MisconfigurationMode,
 }
 
 impl ProjectMetadata {
@@ -47,6 +50,7 @@ impl ProjectMetadata {
             root,
             extra_configuration_paths: Vec::default(),
             options: Options::default(),
+            misconfiguration_mode: MisconfigurationMode::Fail,
         }
     }
 
@@ -70,6 +74,7 @@ impl ProjectMetadata {
             root: system.current_directory().to_path_buf(),
             options,
             extra_configuration_paths: vec![path],
+            misconfiguration_mode: MisconfigurationMode::Fail,
         })
     }
 
@@ -82,6 +87,7 @@ impl ProjectMetadata {
             pyproject.tool.and_then(|tool| tool.ty).unwrap_or_default(),
             root,
             pyproject.project.as_ref(),
+            MisconfigurationMode::Fail,
         )
     }
 
@@ -90,6 +96,7 @@ impl ProjectMetadata {
         mut options: Options,
         root: SystemPathBuf,
         project: Option<&Project>,
+        misconfiguration_mode: MisconfigurationMode,
     ) -> Result<Self, ResolveRequiresPythonError> {
         let name = project
             .and_then(|project| project.name.as_deref())
@@ -117,6 +124,7 @@ impl ProjectMetadata {
             root,
             options,
             extra_configuration_paths: Vec::new(),
+            misconfiguration_mode,
         })
     }
 
@@ -194,6 +202,7 @@ impl ProjectMetadata {
                     pyproject
                         .as_ref()
                         .and_then(|pyproject| pyproject.project.as_ref()),
+                    MisconfigurationMode::Fail,
                 )
                 .map_err(|err| {
                     ProjectMetadataError::InvalidRequiresPythonConstraint {
@@ -273,8 +282,13 @@ impl ProjectMetadata {
         system: &dyn System,
         vendored: &VendoredFileSystem,
     ) -> anyhow::Result<ProgramSettings> {
-        self.options
-            .to_program_settings(self.root(), self.name(), system, vendored)
+        self.options.to_program_settings(
+            self.root(),
+            self.name(),
+            system,
+            vendored,
+            self.misconfiguration_mode,
+        )
     }
 
     pub fn apply_overrides(&mut self, overrides: &ProjectOptionsOverrides) {

crates/ty_project/src/metadata/options.rs

@@ -30,9 +30,9 @@ use thiserror::Error;
 use ty_combine::Combine;
 use ty_python_semantic::lint::{Level, LintSource, RuleSelection};
 use ty_python_semantic::{
-    ProgramSettings, PythonEnvironment, PythonPlatform, PythonVersionFileSource,
-    PythonVersionSource, PythonVersionWithSource, SearchPathSettings, SearchPathValidationError,
-    SearchPaths, SitePackagesPaths, SysPrefixPathOrigin,
+    MisconfigurationMode, ProgramSettings, PythonEnvironment, PythonPlatform,
+    PythonVersionFileSource, PythonVersionSource, PythonVersionWithSource, SearchPathSettings,
+    SearchPathValidationError, SearchPaths, SitePackagesPaths, SysPrefixPathOrigin,
 };
 use ty_static::EnvVars;
 
@@ -117,6 +117,7 @@ impl Options {
         project_name: &str,
         system: &dyn System,
         vendored: &VendoredFileSystem,
+        misconfiguration_mode: MisconfigurationMode,
     ) -> anyhow::Result<ProgramSettings> {
         let environment = self.environment.or_default();
 
@@ -154,14 +155,25 @@ impl Options {
                 ValueSource::Editor => SysPrefixPathOrigin::Editor,
             };
 
-            Some(PythonEnvironment::new(
-                python_path.absolute(project_root, system),
-                origin,
-                system,
-            )?)
+            PythonEnvironment::new(python_path.absolute(project_root, system), origin, system)
+                .map_err(anyhow::Error::from)
+                .map(Some)
         } else {
             PythonEnvironment::discover(project_root, system)
-                .context("Failed to discover local Python environment")?
+                .context("Failed to discover local Python environment")
+        };
+
+        // If in safe-mode, fallback to None if this fails instead of erroring.
+        let python_environment = match python_environment {
+            Ok(python_environment) => python_environment,
+            Err(err) => {
+                if misconfiguration_mode == MisconfigurationMode::UseDefault {
+                    tracing::debug!("Default settings failed to discover local Python environment");
+                    None
+                } else {
+                    return Err(err);
+                }
+            }
         };
 
         let self_site_packages = self_environment_search_paths(
@@ -174,11 +186,23 @@ impl Options {
         .unwrap_or_default();
 
         let site_packages_paths = if let Some(python_environment) = python_environment.as_ref() {
-            self_site_packages.concatenate(
-                python_environment
-                    .site_packages_paths(system)
-                    .context("Failed to discover the site-packages directory")?,
-            )
+            let site_packages_paths = python_environment
+                .site_packages_paths(system)
+                .context("Failed to discover the site-packages directory");
+            let site_packages_paths = match site_packages_paths {
+                Ok(paths) => paths,
+                Err(err) => {
+                    if misconfiguration_mode == MisconfigurationMode::UseDefault {
+                        tracing::debug!(
+                            "Default settings failed to discover site-packages directory"
+                        );
+                        SitePackagesPaths::default()
+                    } else {
+                        return Err(err);
+                    }
+                }
+            };
+            self_site_packages.concatenate(site_packages_paths)
         } else {
             tracing::debug!("No virtual environment found");
             self_site_packages
@@ -201,13 +225,15 @@ impl Options {
             .or_else(|| site_packages_paths.python_version_from_layout())
             .unwrap_or_default();
 
+        // Safe mode is handled inside this function, so we just assume this can't fail
         let search_paths = self.to_search_paths(
             project_root,
             project_name,
             site_packages_paths,
             real_stdlib_path,
             system,
             vendored,
+            misconfiguration_mode,
         )?;
 
         tracing::info!(
@@ -222,6 +248,7 @@ impl Options {
         })
     }
 
+    #[expect(clippy::too_many_arguments)]
     fn to_search_paths(
         &self,
         project_root: &SystemPath,
@@ -230,6 +257,7 @@ impl Options {
         real_stdlib_path: Option<SystemPathBuf>,
         system: &dyn System,
         vendored: &VendoredFileSystem,
+        misconfiguration_mode: MisconfigurationMode,
     ) -> Result<SearchPaths, SearchPathValidationError> {
         let environment = self.environment.or_default();
         let src = self.src.or_default();
@@ -344,6 +372,7 @@ impl Options {
                 .map(|path| path.absolute(project_root, system)),
             site_packages_paths: site_packages_paths.into_vec(),
             real_stdlib_path,
+            misconfiguration_mode,
         };
 
         settings.to_search_paths(system, vendored)

crates/ty_python_semantic/src/lib.rs

@@ -17,7 +17,7 @@ pub use module_resolver::{
     resolve_real_module_confident, resolve_real_shadowable_module, system_module_search_paths,
 };
 pub use program::{
-    Program, ProgramSettings, PythonVersionFileSource, PythonVersionSource,
+    MisconfigurationMode, Program, ProgramSettings, PythonVersionFileSource, PythonVersionSource,
     PythonVersionWithSource, SearchPathSettings,
 };
 pub use python_platform::PythonPlatform;