diff --git a/Cargo.lock b/Cargo.lock index 63833a3c..4a39d59e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,6 +224,7 @@ dependencies = [ "lazy_static", "log", "pet-core", + "pet-reporter", "pet-utils", "regex", "serde", @@ -313,6 +314,7 @@ dependencies = [ "log", "pet-conda", "pet-core", + "pet-reporter", "pet-utils", "regex", "serde", diff --git a/crates/pet-conda/Cargo.toml b/crates/pet-conda/Cargo.toml index 69ae0395..7c4a73da 100644 --- a/crates/pet-conda/Cargo.toml +++ b/crates/pet-conda/Cargo.toml @@ -12,5 +12,9 @@ pet-utils = { path = "../pet-utils" } log = "0.4.21" regex = "1.10.4" +[dev-dependencies] +pet-reporter = { path = "../pet-reporter" } + + [features] ci = [] \ No newline at end of file diff --git a/crates/pet-conda/src/lib.rs b/crates/pet-conda/src/lib.rs index 523c1b46..f412cdad 100644 --- a/crates/pet-conda/src/lib.rs +++ b/crates/pet-conda/src/lib.rs @@ -7,7 +7,8 @@ use environments::{get_conda_environment_info, CondaEnvironment}; use log::error; use manager::CondaManager; use pet_core::{ - os_environment::Environment, python_environment::PythonEnvironment, Locator, LocatorResult, + os_environment::Environment, python_environment::PythonEnvironment, reporter::Reporter, + Locator, LocatorResult, }; use pet_utils::env::PythonEnv; use std::{ @@ -147,73 +148,86 @@ impl Locator for Conda { None } - fn find(&self) -> Option { - // 1. Get a list of all know conda environments - let known_conda_envs = - get_conda_environments(&get_conda_environment_paths(&self.env_vars), &None); - let mut new_managers = vec![]; - { - let mut managers = self.managers.lock().unwrap(); - // 2. Go through all conda dirs and build the conda managers. - for env in &known_conda_envs { - if let Some(conda_dir) = &env.conda_dir { - if managers.contains_key(conda_dir) { - continue; + fn find(&self, reporter: &dyn Reporter) { + let env_vars = self.env_vars.clone(); + thread::scope(|s| { + // 1. Get a list of all know conda environments file paths + let possible_conda_envs = get_conda_environment_paths(&env_vars); + for path in possible_conda_envs { + s.spawn(move || { + // 2. Get the details of the conda environment + // This we do not get any details, then its not a conda environment + let env = get_conda_environment_info(&path, &None)?; + + // 3. If we have a conda environment without a conda_dir + // Then we will not be able to get the manager. + // Either way report this environment + if env.conda_dir.is_none(){ + // We will still return the conda env even though we do not have the manager. + // This might seem incorrect, however the tool is about discovering environments. + // The client can activate this env either using another conda manager or using the activation scripts + error!("Unable to find Conda Manager for the Conda env: {:?}", env); + let prefix = env.prefix.clone(); + let env = env.to_python_environment(None, None); + let mut environments = self.environments.lock().unwrap(); + environments.insert(prefix, env.clone()); + reporter.report_environment(&env); + return None; } - if let Some(manager) = CondaManager::from(conda_dir) { - new_managers.push(manager.to_manager()); - managers.insert(conda_dir.clone(), manager); + + // 3. We have a conda environment with a conda_dir (above we handled the case when its not found) + // We will try to get the manager for this conda_dir + let prefix = env.clone().prefix.clone(); + + { + // 3.1 Check if we have already reported this environment. + // Closure to quickly release lock + let environments = self.environments.lock().unwrap(); + if environments.contains_key(&env.prefix) { + return None; + } } - } - } - } - let mut environments = self.environments.lock().unwrap(); - let mut new_environments: Vec = vec![]; - // 3. Go through each environment we know of and build the python environments. - for known_env in &known_conda_envs { - if environments.contains_key(&known_env.prefix) { - continue; - } - if let Some(conda_dir) = &known_env.conda_dir { - if let Some(manager) = self.get_manager(conda_dir) { - let env = known_env.to_python_environment( - Some(manager.conda_dir.clone()), - Some(manager.to_manager()), - ); - environments.insert(known_env.prefix.clone(), env.clone()); - new_environments.push(env); - } else { - // We will still return the conda env even though we do not have the manager. - // This might seem incorrect, however the tool is about discovering environments. - // The client can activate this env either using another conda manager or using the activation scripts - error!("Unable to find Conda Manager for Conda env (even though we have a conda_dir {:?}): Env Details = {:?}", conda_dir, known_env); - let env = known_env.to_python_environment(Some(conda_dir.clone()), None); - environments.insert(known_env.prefix.clone(), env.clone()); - new_environments.push(env); - } - } else { - // We will still return the conda env even though we do not have the manager. - // This might seem incorrect, however the tool is about discovering environments. - // The client can activate this env either using another conda manager or using the activation scripts - error!( - "Unable to find Conda Manager for the Conda env: {:?}", - known_env - ); - let env = known_env.to_python_environment(None, None); - environments.insert(known_env.prefix.clone(), env.clone()); - new_environments.push(env); - } - } - if new_managers.is_empty() && new_environments.is_empty() { - return None; - } + // 4 Get the manager for this env. + let conda_dir = &env.conda_dir.clone()?; + let managers = self.managers.lock().unwrap(); + let mut manager = managers.get(conda_dir).cloned(); + drop(managers); - Some(LocatorResult { - managers: new_managers, - environments: new_environments, - }) + if manager.is_none() { + // 4.1 Build the manager from the conda dir if we do not have it. + if let Some(conda_manager) = CondaManager::from(conda_dir) { + reporter.report_manager(&conda_manager.to_manager()); + let mut managers = self.managers.lock().unwrap(); + managers.insert(conda_dir.to_path_buf().clone(), conda_manager.clone()); + manager = Some(conda_manager); + } + } + + // 5. Report this env. + if let Some(manager) = manager { + let env = env.to_python_environment( + Some(manager.conda_dir.clone()), + Some(manager.to_manager()), + ); + let mut environments = self.environments.lock().unwrap(); + environments.insert(prefix.clone(), env.clone()); + reporter.report_environment(&env); + } else { + // We will still return the conda env even though we do not have the manager. + // This might seem incorrect, however the tool is about discovering environments. + // The client can activate this env either using another conda manager or using the activation scripts + error!("Unable to find Conda Manager for Conda env (even though we have a conda_dir {:?}): Env Details = {:?}", conda_dir, env); + let env = env.to_python_environment(Some(conda_dir.clone()), None); + let mut environments = self.environments.lock().unwrap(); + environments.insert(prefix.clone(), env.clone()); + reporter.report_environment(&env); + } + Option::<()>::Some(()) + }); + } + }); } } diff --git a/crates/pet-conda/tests/ci_test.rs b/crates/pet-conda/tests/ci_test.rs index c8a7c7e2..7bc22f48 100644 --- a/crates/pet-conda/tests/ci_test.rs +++ b/crates/pet-conda/tests/ci_test.rs @@ -17,12 +17,14 @@ fn detect_conda_root() { manager::EnvManagerType, os_environment::EnvironmentApi, python_environment::PythonEnvironmentCategory, Locator, }; - use std::path::PathBuf; + use pet_reporter::test::create_reporter; let env = EnvironmentApi::new(); + let reporter = create_reporter(); let conda = Conda::from(&env); - let result = conda.find().unwrap(); + conda.find(&reporter); + let result = reporter.get_result(); assert_eq!(result.managers.len(), 1); @@ -92,6 +94,7 @@ fn detect_new_conda_env() { use pet_core::{ os_environment::EnvironmentApi, python_environment::PythonEnvironmentCategory, Locator, }; + use pet_reporter::test::create_reporter; use std::path::PathBuf; let env_name = "env_with_python"; @@ -102,7 +105,9 @@ fn detect_new_conda_env() { let env = EnvironmentApi::new(); let conda = Conda::from(&env); - let result = conda.find().unwrap(); + let reporter = create_reporter(); + conda.find(&reporter); + let result = reporter.get_result(); assert_eq!(result.managers.len(), 1); @@ -191,6 +196,7 @@ fn detect_new_conda_env_without_python() { use pet_core::{ os_environment::EnvironmentApi, python_environment::PythonEnvironmentCategory, Locator, }; + use pet_reporter::test::create_reporter; use std::path::PathBuf; let env_name = "env_without_python"; @@ -198,7 +204,9 @@ fn detect_new_conda_env_without_python() { let env = EnvironmentApi::new(); let conda = Conda::from(&env); - let result = conda.find().unwrap(); + let reporter = create_reporter(); + conda.find(&reporter); + let result = reporter.get_result(); assert_eq!(result.managers.len(), 1); @@ -238,6 +246,7 @@ fn detect_new_conda_env_created_with_p_flag_without_python() { use pet_core::{ os_environment::EnvironmentApi, python_environment::PythonEnvironmentCategory, Locator, }; + use pet_reporter::test::create_reporter; use std::path::PathBuf; let env_name = "env_without_python3"; @@ -246,7 +255,9 @@ fn detect_new_conda_env_created_with_p_flag_without_python() { let env = EnvironmentApi::new(); let conda = Conda::from(&env); - let result = conda.find().unwrap(); + let reporter = create_reporter(); + conda.find(&reporter); + let result = reporter.get_result(); assert_eq!(result.managers.len(), 1); @@ -286,6 +297,7 @@ fn detect_new_conda_env_created_with_p_flag_with_python() { use pet_core::{ os_environment::EnvironmentApi, python_environment::PythonEnvironmentCategory, Locator, }; + use pet_reporter::test::create_reporter; use std::path::PathBuf; let env_name = "env_with_python3"; @@ -298,7 +310,9 @@ fn detect_new_conda_env_created_with_p_flag_with_python() { let env = EnvironmentApi::new(); let conda = Conda::from(&env); - let result = conda.find().unwrap(); + let reporter = create_reporter(); + conda.find(&reporter); + let result = reporter.get_result(); assert_eq!(result.managers.len(), 1); diff --git a/crates/pet-core/src/lib.rs b/crates/pet-core/src/lib.rs index dee0d930..fbc7dc93 100644 --- a/crates/pet-core/src/lib.rs +++ b/crates/pet-core/src/lib.rs @@ -4,6 +4,7 @@ use manager::EnvManager; use pet_utils::env::PythonEnv; use python_environment::PythonEnvironment; +use reporter::Reporter; pub mod arch; pub mod manager; @@ -39,5 +40,5 @@ pub trait Locator: Send + Sync { /** * Finds all environments specific to this locator. */ - fn find(&self) -> Option; + fn find(&self, reporter: &dyn Reporter); } diff --git a/crates/pet-env-var-path/src/lib.rs b/crates/pet-env-var-path/src/lib.rs index 97a17dbc..22179122 100644 --- a/crates/pet-env-var-path/src/lib.rs +++ b/crates/pet-env-var-path/src/lib.rs @@ -1,15 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -use std::{cell::RefCell, collections::HashMap, path::PathBuf, thread}; - use env_variables::EnvVariables; use lazy_static::lazy_static; use log::warn; use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory}, - Locator, LocatorResult, + reporter::Reporter, + Locator, }; use pet_utils::{ env::PythonEnv, @@ -19,6 +18,7 @@ use pet_utils::{ pyvenv_cfg::PyVenvCfg, }; use regex::Regex; +use std::{cell::RefCell, collections::HashMap, path::PathBuf, thread}; lazy_static! { // /Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10 @@ -81,134 +81,132 @@ impl Locator for PythonOnPath { Some(env) } - fn find(&self) -> Option { - // Exclude files from this folder, as they would have been discovered elsewhere (widows_store) - // Also the exe is merely a pointer to another file. - let home = self.env_vars.home.clone()?; - let apps_path = home - .join("AppData") - .join("Local") - .join("Microsoft") - .join("WindowsApps"); - - let items = self - .env_vars - .known_global_search_locations - .clone() - .into_iter() - .filter(|p| !p.starts_with(apps_path.clone())) - .collect::>(); - - let mut handles: Vec>> = vec![]; - for item in items.chunks(5) { - let lst = item.to_vec(); - let handle = thread::spawn(move || { - lst.iter() - // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. - // Hence do not just look for files in a bin directory of the path. - .flat_map(|p| find_executables(p)) - .filter(|p| { - // Exclude python2 on macOS - if std::env::consts::OS == "macos" { - return p.to_str().unwrap_or_default() != "/usr/bin/python2"; - } - true - }) - .collect::>() - }); - handles.push(handle); - } - let mut python_executables: Vec = vec![]; - for handle in handles { - if let Ok(ref mut result) = handle.join() { - python_executables.append(result) + fn find(&self, reporter: &dyn Reporter) { + if let Some(home) = &self.env_vars.home { + // Exclude files from this folder, as they would have been discovered elsewhere (widows_store) + // Also the exe is merely a pointer to another file. + let apps_path = home + .join("AppData") + .join("Local") + .join("Microsoft") + .join("WindowsApps"); + + let items = self + .env_vars + .known_global_search_locations + .clone() + .into_iter() + .filter(|p| !p.starts_with(apps_path.clone())) + .collect::>(); + + let mut handles: Vec>> = vec![]; + for item in items.chunks(5) { + let lst = item.to_vec(); + let handle = thread::spawn(move || { + lst.iter() + // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. + // Hence do not just look for files in a bin directory of the path. + .flat_map(|p| find_executables(p)) + .filter(|p| { + // Exclude python2 on macOS + if std::env::consts::OS == "macos" { + return p.to_str().unwrap_or_default() != "/usr/bin/python2"; + } + true + }) + .collect::>() + }); + handles.push(handle); + } + let mut python_executables: Vec = vec![]; + for handle in handles { + if let Ok(ref mut result) = handle.join() { + python_executables.append(result) + } } - } - // The python executables can contain files like - // /usr/local/bin/python3.10 - // /usr/local/bin/python3 - // Possible both of the above are symlinks and point to the same file. - // Hence sort on length of the path. - // So that we process generic python3 before python3.10 - python_executables.sort(); - python_executables.dedup(); - python_executables.sort_by(|a, b| { - a.to_str() - .unwrap_or_default() - .len() - .cmp(&b.to_str().unwrap_or_default().len()) - }); - - let mut already_found: HashMap> = HashMap::new(); - python_executables.into_iter().for_each(|exe| { - if let Some(exe_dir) = exe.parent() { - let mut version = None; - let symlink = match PyVenvCfg::find(exe_dir) { - Some(version_value) => { - version = Some(version_value.version); - // We got a version from pyvenv.cfg file, that means we're looking at a virtual env. - // This should not happen. - warn!( - "Found a virtual env but identified as global Python: {:?}", - exe - ); - // Its already fully resolved as we managed to get the env version from a pyvenv.cfg in current dir. - None - } - None => resolve_symlink(&exe.clone()), - }; - if let Some(ref symlink) = symlink { - if already_found.contains_key(symlink) { - // If we have a symlinked file then, ensure the original path is added as symlink. - // Possible we only added /usr/local/bin/python3.10 and not /usr/local/bin/python3 - // This entry is /usr/local/bin/python3 - if let Some(existing) = already_found.get_mut(&exe) { - let mut existing = existing.borrow_mut(); - if let Some(ref mut symlinks) = existing.symlinks { - symlinks.push(exe.clone()); - } else { - existing.symlinks = Some(vec![symlink.clone(), exe.clone()]); - } + // The python executables can contain files like + // /usr/local/bin/python3.10 + // /usr/local/bin/python3 + // Possible both of the above are symlinks and point to the same file. + // Hence sort on length of the path. + // So that we process generic python3 before python3.10 + python_executables.sort(); + python_executables.dedup(); + python_executables.sort_by(|a, b| { + a.to_str() + .unwrap_or_default() + .len() + .cmp(&b.to_str().unwrap_or_default().len()) + }); - if let Some(shortest_exe) = get_shortest_executable(&existing.symlinks) - { - existing.executable = Some(shortest_exe); + let mut already_found: HashMap> = HashMap::new(); + python_executables.into_iter().for_each(|exe| { + if let Some(exe_dir) = exe.parent() { + let mut version = None; + let symlink = match PyVenvCfg::find(exe_dir) { + Some(version_value) => { + version = Some(version_value.version); + // We got a version from pyvenv.cfg file, that means we're looking at a virtual env. + // This should not happen. + warn!( + "Found a virtual env but identified as global Python: {:?}", + exe + ); + // Its already fully resolved as we managed to get the env version from a pyvenv.cfg in current dir. + None + } + None => resolve_symlink(&exe.clone()), + }; + if let Some(ref symlink) = symlink { + if already_found.contains_key(symlink) { + // If we have a symlinked file then, ensure the original path is added as symlink. + // Possible we only added /usr/local/bin/python3.10 and not /usr/local/bin/python3 + // This entry is /usr/local/bin/python3 + if let Some(existing) = already_found.get_mut(&exe) { + let mut existing = existing.borrow_mut(); + if let Some(ref mut symlinks) = existing.symlinks { + symlinks.push(exe.clone()); + } else { + existing.symlinks = Some(vec![symlink.clone(), exe.clone()]); + } + + if let Some(shortest_exe) = + get_shortest_executable(&existing.symlinks) + { + existing.executable = Some(shortest_exe); + } } + return; } - return; } - } - if let Some(env) = self.from(&PythonEnv::new(exe.clone(), None, version)) { - let mut env = env.clone(); - let mut symlinks: Option> = None; - if let Some(ref symlink) = symlink { - symlinks = Some(vec![symlink.clone(), exe.clone()]); - } - env.symlinks.clone_from(&symlinks); - if let Some(shortest_exe) = get_shortest_executable(&symlinks) { - env.executable = Some(shortest_exe); - } + if let Some(env) = self.from(&PythonEnv::new(exe.clone(), None, version)) { + let mut env = env.clone(); + let mut symlinks: Option> = None; + if let Some(ref symlink) = symlink { + symlinks = Some(vec![symlink.clone(), exe.clone()]); + } + env.symlinks.clone_from(&symlinks); + if let Some(shortest_exe) = get_shortest_executable(&symlinks) { + env.executable = Some(shortest_exe); + } - let env = RefCell::new(env); - already_found.insert(exe, env.clone()); - if let Some(symlinks) = symlinks.clone() { - for symlink in symlinks { - already_found.insert(symlink.clone(), env.clone()); + let env = RefCell::new(env); + already_found.insert(exe, env.clone()); + if let Some(symlinks) = symlinks.clone() { + for symlink in symlinks { + already_found.insert(symlink.clone(), env.clone()); + } } } } - } - }); - - if already_found.is_empty() { - None - } else { - Some(LocatorResult { - environments: already_found.values().map(|v| v.borrow().clone()).collect(), - managers: vec![], - }) + }); + + already_found + .values() + .map(|v| v.borrow().clone()) + .for_each(|e| reporter.report_environment(&e)); } } } diff --git a/crates/pet-homebrew/src/lib.rs b/crates/pet-homebrew/src/lib.rs index 6eecd7d0..1eb9ef3f 100644 --- a/crates/pet-homebrew/src/lib.rs +++ b/crates/pet-homebrew/src/lib.rs @@ -5,7 +5,7 @@ use env_variables::EnvVariables; use environment_locations::get_homebrew_prefix_bin; use environments::get_python_info; use pet_core::{ - os_environment::Environment, python_environment::PythonEnvironment, Locator, LocatorResult, + os_environment::Environment, python_environment::PythonEnvironment, reporter::Reporter, Locator, }; use pet_utils::{env::PythonEnv, executable::find_executables, path::resolve_symlink}; use std::{collections::HashSet, path::PathBuf}; @@ -80,9 +80,8 @@ impl Locator for Homebrew { resolve(env, &mut reported) } - fn find(&self) -> Option { + fn find(&self, reporter: &dyn Reporter) { let mut reported: HashSet = HashSet::new(); - let mut environments: Vec = vec![]; for homebrew_prefix_bin in get_homebrew_prefix_bin(&self.environment) { for file in find_executables(&homebrew_prefix_bin).iter().filter(|f| { let file_name = f @@ -104,7 +103,7 @@ impl Locator for Homebrew { // Hence call `resolve` to correctly identify homebrew python installs. let env_to_resolve = PythonEnv::new(file.clone(), None, None); if let Some(env) = resolve(&env_to_resolve, &mut reported) { - environments.push(env); + reporter.report_environment(&env); } } @@ -113,17 +112,8 @@ impl Locator for Homebrew { let file = homebrew_prefix_bin.join("python3"); let env_to_resolve = PythonEnv::new(file, None, None); if let Some(env) = resolve(&env_to_resolve, &mut reported) { - environments.push(env); + reporter.report_environment(&env); } } - - if environments.is_empty() { - None - } else { - Some(LocatorResult { - managers: vec![], - environments, - }) - } } } diff --git a/crates/pet-pipenv/src/lib.rs b/crates/pet-pipenv/src/lib.rs index 19769567..c1bddbc1 100644 --- a/crates/pet-pipenv/src/lib.rs +++ b/crates/pet-pipenv/src/lib.rs @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use std::{fs, path::PathBuf}; - use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory}, - Locator, LocatorResult, + reporter::Reporter, + Locator, }; use pet_utils::{env::PythonEnv, path::normalize}; +use std::{fs, path::PathBuf}; fn get_pipenv_project(env: &PythonEnv) -> Option { let project_file = env.prefix.clone()?.join(".project"); @@ -58,7 +58,7 @@ impl Locator for PipEnv { ) } - fn find(&self) -> Option { - None + fn find(&self, _reporter: &dyn Reporter) { + // } } diff --git a/crates/pet-pyenv/Cargo.toml b/crates/pet-pyenv/Cargo.toml index 8c7fa1ec..b2d6cde5 100644 --- a/crates/pet-pyenv/Cargo.toml +++ b/crates/pet-pyenv/Cargo.toml @@ -12,3 +12,6 @@ pet-utils = { path = "../pet-utils" } pet-conda = { path = "../pet-conda" } log = "0.4.21" regex = "1.10.4" + +[dev-dependencies] +pet-reporter = { path = "../pet-reporter" } diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index 64b8abdb..66fe4ec9 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -13,7 +13,8 @@ use pet_core::{ manager::{EnvManager, EnvManagerType}, os_environment::Environment, python_environment::PythonEnvironment, - Locator, LocatorResult, + reporter::Reporter, + Locator, }; use pet_utils::env::PythonEnv; @@ -66,34 +67,24 @@ impl Locator for PyEnv { None } - fn find(&self) -> Option { + fn find(&self, reporter: &dyn Reporter) { let pyenv_info = PyEnvInfo::from(&self.env_vars); - let mut managers: Vec = vec![]; let mut manager: Option = None; - let mut environments: Vec = vec![]; if let Some(ref exe) = pyenv_info.exe { let version = pyenv_info.version.clone(); - manager = Some(EnvManager::new(exe.clone(), EnvManagerType::Pyenv, version)); - managers.push(manager.clone().unwrap()); + let mgr = EnvManager::new(exe.clone(), EnvManagerType::Pyenv, version); + reporter.report_manager(&mgr); + manager = Some(mgr); } if let Some(ref versions) = &pyenv_info.versions { if let Some(envs) = list_pyenv_environments(&manager, versions, &self.conda_locator) { for env in envs.environments { - environments.push(env); + reporter.report_environment(&env); } for mgr in envs.managers { - managers.push(mgr); + reporter.report_manager(&mgr); } } } - - if environments.is_empty() && managers.is_empty() { - None - } else { - Some(LocatorResult { - managers, - environments, - }) - } } } diff --git a/crates/pet-pyenv/tests/pyenv_test.rs b/crates/pet-pyenv/tests/pyenv_test.rs index bb95caa1..a164732d 100644 --- a/crates/pet-pyenv/tests/pyenv_test.rs +++ b/crates/pet-pyenv/tests/pyenv_test.rs @@ -11,6 +11,7 @@ fn does_not_find_any_pyenv_envs() { use pet_core::{self, Locator}; use pet_pyenv; use pet_pyenv::PyEnv; + use pet_reporter::test::create_reporter; use std::{collections::HashMap, path::PathBuf, sync::Arc}; let environment = create_test_environment( @@ -22,9 +23,12 @@ fn does_not_find_any_pyenv_envs() { let conda = Arc::new(Conda::from(&environment)); let locator = PyEnv::from(&environment, conda); - let result = locator.find(); + let reporter = create_reporter(); + locator.find(&reporter); + let result = reporter.get_result(); - assert_eq!(result.is_none(), true); + assert_eq!(result.managers.is_empty(), true); + assert_eq!(result.environments.is_empty(), true); } #[test] @@ -40,6 +44,7 @@ fn does_not_find_any_pyenv_envs_even_with_pyenv_installed() { }; use pet_pyenv; use pet_pyenv::PyEnv; + use pet_reporter::test::create_reporter; use serde_json::json; use std::{collections::HashMap, path::PathBuf, sync::Arc}; @@ -62,7 +67,9 @@ fn does_not_find_any_pyenv_envs_even_with_pyenv_installed() { let conda = Arc::new(Conda::from(&environment)); let locator = PyEnv::from(&environment, conda); - let result = locator.find().unwrap(); + let reporter = create_reporter(); + locator.find(&reporter); + let result = reporter.get_result(); let managers = result.clone().managers; assert_eq!(managers.len(), 1); @@ -90,6 +97,7 @@ fn find_pyenv_envs() { }; use pet_pyenv; use pet_pyenv::PyEnv; + use pet_reporter::test::create_reporter; use std::{collections::HashMap, path::PathBuf, sync::Arc}; let home = resolve_test_path(&["unix", "pyenv", "user_home"]); @@ -114,7 +122,9 @@ fn find_pyenv_envs() { let conda = Arc::new(Conda::from(&environment)); let locator = PyEnv::from(&environment, conda); - let mut result = locator.find().unwrap(); + let reporter = create_reporter(); + locator.find(&reporter); + let mut result = reporter.get_result(); assert_eq!(result.managers.len(), 2); diff --git a/crates/pet-reporter/src/test.rs b/crates/pet-reporter/src/test.rs index 304dc587..60b1c1b9 100644 --- a/crates/pet-reporter/src/test.rs +++ b/crates/pet-reporter/src/test.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. use crate::environment::get_environment_key; +use pet_core::LocatorResult; use pet_core::{manager::EnvManager, python_environment::PythonEnvironment, reporter::Reporter}; use std::collections::HashMap; use std::{ @@ -10,13 +11,28 @@ use std::{ }; pub struct TestReporter { - pub reported_managers: Arc>>, - pub reported_environments: Arc>>, + managers: Arc>>, + environments: Arc>>, +} + +impl TestReporter { + pub fn get_result(&self) -> LocatorResult { + LocatorResult { + managers: self.managers.lock().unwrap().values().cloned().collect(), + environments: self + .environments + .lock() + .unwrap() + .values() + .cloned() + .collect(), + } + } } impl Reporter for TestReporter { fn report_manager(&self, manager: &EnvManager) { - let mut reported_managers = self.reported_managers.lock().unwrap(); + let mut reported_managers = self.managers.lock().unwrap(); if !reported_managers.contains_key(&manager.executable) { reported_managers.insert(manager.executable.clone(), manager.clone()); } @@ -24,7 +40,7 @@ impl Reporter for TestReporter { fn report_environment(&self, env: &PythonEnvironment) { if let Some(key) = get_environment_key(env) { - let mut reported_environments = self.reported_environments.lock().unwrap(); + let mut reported_environments = self.environments.lock().unwrap(); if !reported_environments.contains_key(key) { reported_environments.insert(key.clone(), env.clone()); } @@ -37,7 +53,7 @@ impl Reporter for TestReporter { pub fn create_reporter() -> TestReporter { TestReporter { - reported_managers: Arc::new(Mutex::new(HashMap::new())), - reported_environments: Arc::new(Mutex::new(HashMap::new())), + managers: Arc::new(Mutex::new(HashMap::new())), + environments: Arc::new(Mutex::new(HashMap::new())), } } diff --git a/crates/pet-venv/src/lib.rs b/crates/pet-venv/src/lib.rs index 6db9e724..2ba6b204 100644 --- a/crates/pet-venv/src/lib.rs +++ b/crates/pet-venv/src/lib.rs @@ -3,7 +3,8 @@ use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory}, - Locator, LocatorResult, + reporter::Reporter, + Locator, }; use pet_utils::{env::PythonEnv, headers::Headers, pyvenv_cfg::PyVenvCfg}; @@ -60,9 +61,8 @@ impl Locator for Venv { } } - fn find(&self) -> Option { + fn find(&self, _reporter: &dyn Reporter) { // There are no common global locations for virtual environments. // We expect the user of this class to call `is_compatible` - None } } diff --git a/crates/pet-virtualenv/src/lib.rs b/crates/pet-virtualenv/src/lib.rs index 17e699b9..f888e303 100644 --- a/crates/pet-virtualenv/src/lib.rs +++ b/crates/pet-virtualenv/src/lib.rs @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use std::fs; - use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory}, - Locator, LocatorResult, + reporter::Reporter, + Locator, }; use pet_utils::{env::PythonEnv, headers::Headers}; +use std::fs; pub fn is_virtualenv(env: &PythonEnv) -> bool { if env.prefix.is_none() { @@ -91,9 +91,8 @@ impl Locator for VirtualEnv { } } - fn find(&self) -> Option { + fn find(&self, _reporter: &dyn Reporter) { // There are no common global locations for virtual environments. // We expect the user of this class to call `is_compatible` - None } } diff --git a/crates/pet-virtualenvwrapper/src/lib.rs b/crates/pet-virtualenvwrapper/src/lib.rs index 1daa2b95..8d069aa0 100644 --- a/crates/pet-virtualenvwrapper/src/lib.rs +++ b/crates/pet-virtualenvwrapper/src/lib.rs @@ -7,7 +7,8 @@ use environments::{get_project, is_virtualenvwrapper, list_python_environments}; use pet_core::{ os_environment::Environment, python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory}, - Locator, LocatorResult, + reporter::Reporter, + Locator, }; use pet_utils::{env::PythonEnv, headers::Headers}; @@ -57,22 +58,15 @@ impl Locator for VirtualEnvWrapper { ) } - fn find(&self) -> Option { - let work_on_home = get_work_on_home_path(&self.env_vars)?; - let envs = list_python_environments(&work_on_home)?; - let mut environments: Vec = vec![]; - envs.iter().for_each(|env| { - if let Some(env) = self.from(env) { - environments.push(env); + fn find(&self, reporter: &dyn Reporter) { + if let Some(work_on_home) = get_work_on_home_path(&self.env_vars) { + if let Some(envs) = list_python_environments(&work_on_home) { + envs.iter().for_each(|env| { + if let Some(env) = self.from(env) { + reporter.report_environment(&env); + } + }); } - }); - if environments.is_empty() { - None - } else { - Some(LocatorResult { - managers: vec![], - environments, - }) } } } diff --git a/crates/pet-windows-registry/src/lib.rs b/crates/pet-windows-registry/src/lib.rs index a1ea29b7..989d8756 100644 --- a/crates/pet-windows-registry/src/lib.rs +++ b/crates/pet-windows-registry/src/lib.rs @@ -4,7 +4,7 @@ #[cfg(windows)] use environments::get_registry_pythons; use pet_conda::{utils::is_conda_env, CondaLocator}; -use pet_core::{python_environment::PythonEnvironment, Locator, LocatorResult}; +use pet_core::{python_environment::PythonEnvironment, reporter::Reporter, Locator}; use pet_utils::env::PythonEnv; use std::sync::Arc; @@ -40,13 +40,21 @@ impl Locator for WindowsRegistry { None } - fn find(&self) -> Option { - #[cfg(windows)] + #[cfg(windows)] + fn find(&self, reporter: &dyn Reporter) { if let Some(result) = get_registry_pythons(&self.conda_locator) { - if !result.environments.is_empty() || !result.managers.is_empty() { - return Some(result); - } + result + .managers + .iter() + .for_each(|m| reporter.report_manager(m)); + result + .environments + .iter() + .for_each(|e| reporter.report_environment(e)); } - None + } + #[cfg(unix)] + fn find(&self, _reporter: &dyn Reporter) { + // } } diff --git a/crates/pet-windows-store/src/lib.rs b/crates/pet-windows-store/src/lib.rs index 4f2292bf..dd0a6f6a 100644 --- a/crates/pet-windows-store/src/lib.rs +++ b/crates/pet-windows-store/src/lib.rs @@ -9,7 +9,8 @@ use crate::env_variables::EnvVariables; #[cfg(windows)] use environments::list_store_pythons; use pet_core::python_environment::PythonEnvironment; -use pet_core::{os_environment::Environment, Locator, LocatorResult}; +use pet_core::reporter::Reporter; +use pet_core::{os_environment::Environment, Locator}; use pet_utils::env::PythonEnv; use std::path::Path; @@ -46,19 +47,15 @@ impl Locator for WindowsStore { None } - fn find(&self) -> Option { - #[cfg(windows)] - let environments = list_store_pythons(&self.env_vars)?; - #[cfg(unix)] - let environments = vec![]; - - if environments.is_empty() { - None - } else { - Some(LocatorResult { - managers: vec![], - environments, - }) + #[cfg(windows)] + fn find(&self, reporter: &dyn Reporter) { + if let Some(items) = list_store_pythons(&self.env_vars) { + items.iter().for_each(|e| reporter.report_environment(e)) } } + + #[cfg(unix)] + fn find(&self, _reporter: &dyn Reporter) { + // + } } diff --git a/crates/pet/src/locators.rs b/crates/pet/src/locators.rs index 5ed0b620..7253e352 100644 --- a/crates/pet/src/locators.rs +++ b/crates/pet/src/locators.rs @@ -4,9 +4,8 @@ use log::{error, info}; use pet_conda::Conda; use pet_core::os_environment::{Environment, EnvironmentApi}; -use pet_core::python_environment::PythonEnvironment; use pet_core::reporter::Reporter; -use pet_core::{Locator, LocatorResult}; +use pet_core::Locator; use pet_global_virtualenvs::list_global_virtual_envs_paths; use pet_pipenv::PipEnv; use pet_pyenv::PyEnv; @@ -17,118 +16,109 @@ use pet_venv::Venv; use pet_virtualenv::VirtualEnv; use pet_virtualenvwrapper::VirtualEnvWrapper; use std::path::PathBuf; -use std::thread::JoinHandle; use std::time::SystemTime; use std::{sync::Arc, thread}; pub fn find_and_report_envs(reporter: &dyn Reporter) { info!("Started Refreshing Environments"); let now = SystemTime::now(); + // let reporter = Arc::new(reporter); // 1. Find using known global locators. - let mut threads = find_using_global_finders(); - - // Step 2: Search in some global locations for virtual envs. - threads.push(thread::spawn(find_in_global_virtual_env_dirs)); - - // Step 3: Finally find in the current PATH variable - // threads.push(thread::spawn(find_in_path_env_variable)); - - // NOTE: Ensure we process the results in the same order as they were started. - // This will ensure the priority order is maintained. - for handle in threads { - match handle.join() { - Ok(result) => report_result(result, reporter), - Err(err) => error!("One of the finders failed. {:?}", err), - } - } + thread::scope(|s| { + s.spawn(|| find_using_global_finders(reporter)); + // Step 2: Search in some global locations for virtual envs. + s.spawn(|| find_in_global_virtual_env_dirs(reporter)); + // Step 3: Finally find in the current PATH variable + // s.spawn(find_in_path_env_variable()); + }); reporter.report_completion(now.elapsed().unwrap_or_default()); } -fn find_using_global_finders(// dispatcher: &mut dyn MessageDispatcher, -) -> Vec>> { +fn find_using_global_finders(reporter: &dyn Reporter) { // Step 1: These environments take precedence over all others. // As they are very specific and guaranteed to be specific type. #[cfg(windows)] - fn find() -> Vec>> { - use pet_windows_registry::WindowsRegistry; - use pet_windows_store::WindowsStore; - // use pet_win - // The order matters, - // Windows store can sometimes get detected via registry locator (but we want to avoid that), - // difficult to repro, but we have see this on Karthiks machine - // Windows registry can contain conda envs (e.g. installing Ananconda will result in registry entries). - // Conda is best done last, as Windows Registry and Pyenv can also contain conda envs, - // Thus lets leave the generic conda locator to last to find all remaining conda envs. - // pyenv can be treated as a virtualenvwrapper environment, hence virtualenvwrapper needs to be detected first - let environment = EnvironmentApi::new(); - let conda_locator = Arc::new(Conda::from(&environment)); - let conda_locator1 = conda_locator.clone(); - let conda_locator2 = conda_locator.clone(); - let conda_locator3 = conda_locator.clone(); - vec![ + fn find(reporter: &dyn Reporter) { + thread::scope(|s| { + use pet_windows_registry::WindowsRegistry; + use pet_windows_store::WindowsStore; + // use pet_win + // The order matters, + // Windows store can sometimes get detected via registry locator (but we want to avoid that), + // difficult to repro, but we have see this on Karthiks machine + // Windows registry can contain conda envs (e.g. installing Ananconda will result in registry entries). + // Conda is best done last, as Windows Registry and Pyenv can also contain conda envs, + // Thus lets leave the generic conda locator to last to find all remaining conda envs. + // pyenv can be treated as a virtualenvwrapper environment, hence virtualenvwrapper needs to be detected first + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + let conda_locator1 = conda_locator.clone(); + let conda_locator2 = conda_locator.clone(); + let conda_locator3 = conda_locator.clone(); + // 1. windows store - thread::spawn(|| { + s.spawn(|| { let environment = EnvironmentApi::new(); - WindowsStore::from(&environment).find() - }), + WindowsStore::from(&environment).find(reporter) + }); // 2. windows registry - thread::spawn(|| WindowsRegistry::from(conda_locator1).find()), + s.spawn(|| WindowsRegistry::from(conda_locator1).find(reporter)); // 3. virtualenvwrapper - thread::spawn(|| { + s.spawn(|| { let environment = EnvironmentApi::new(); - VirtualEnvWrapper::from(&environment).find() - }), + VirtualEnvWrapper::from(&environment).find(reporter) + }); // 4. pyenv - thread::spawn(|| { + s.spawn(|| { let environment = EnvironmentApi::new(); - PyEnv::from(&environment, conda_locator2).find() - }), + PyEnv::from(&environment, conda_locator2).find(reporter) + }); // 5. conda - thread::spawn(move || conda_locator3.find()), - ] + s.spawn(move || conda_locator3.find(reporter)); + }); } #[cfg(unix)] - fn find() -> Vec>> { - // The order matters, - // pyenv can be treated as a virtualenvwrapper environment, hence virtualenvwrapper needs to be detected first - // Homebrew can happen anytime - // Conda is best done last, as pyenv can also contain conda envs, - // Thus lets leave the generic conda locator to last to find all remaining conda envs. - - use pet_homebrew::Homebrew; - - let environment = EnvironmentApi::new(); - let conda_locator = Arc::new(Conda::from(&environment)); - let conda_locator1 = conda_locator.clone(); - let conda_locator2 = conda_locator.clone(); - vec![ + fn find(reporter: &dyn Reporter) { + thread::scope(|s| { + // The order matters, + // pyenv can be treated as a virtualenvwrapper environment, hence virtualenvwrapper needs to be detected first + // Homebrew can happen anytime + // Conda is best done last, as pyenv can also contain conda envs, + // Thus lets leave the generic conda locator to last to find all remaining conda envs. + + use pet_homebrew::Homebrew; + + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + let conda_locator1 = conda_locator.clone(); + let conda_locator2 = conda_locator.clone(); // 1. virtualenvwrapper - thread::spawn(|| { + s.spawn(|| { let environment = EnvironmentApi::new(); - VirtualEnvWrapper::from(&environment).find() - }), + VirtualEnvWrapper::from(&environment).find(reporter) + }); // 2. pyenv - thread::spawn(|| { + s.spawn(|| { let environment = EnvironmentApi::new(); - PyEnv::from(&environment, conda_locator1).find() - }), + PyEnv::from(&environment, conda_locator1).find(reporter) + }); // 3. homebrew - thread::spawn(|| { + s.spawn(|| { let environment = EnvironmentApi::new(); - Homebrew::from(&environment).find() - }), + Homebrew::from(&environment).find(reporter) + }); // 4. conda - thread::spawn(move || conda_locator2.find()), - ] + s.spawn(move || conda_locator2.find(reporter)); + }); } - find() + find(reporter) } -fn find_in_global_virtual_env_dirs() -> Option { +fn find_in_global_virtual_env_dirs(reporter: &dyn Reporter) { #[cfg(unix)] use pet_homebrew::Homebrew; @@ -162,7 +152,6 @@ fn find_in_global_virtual_env_dirs() -> Option { ] .concat(); // Step 2: Search in some global locations for virtual envs. - let mut environments = Vec::::new(); for env_path in envs_from_global_locations { if let Some(executable) = find_executable(&env_path) { let mut env = PythonEnv::new(executable.clone(), Some(env_path.clone()), None); @@ -173,7 +162,7 @@ fn find_in_global_virtual_env_dirs() -> Option { // 1. First must be homebrew, as it is the most specific and supports symlinks #[cfg(unix)] if let Some(env) = homebrew_locator.from(&env) { - environments.push(env); + reporter.report_environment(&env); continue; } @@ -186,7 +175,7 @@ fn find_in_global_virtual_env_dirs() -> Option { let mut found = false; for locator in &venv_type_locators { if let Some(env) = locator.as_ref().from(&env) { - environments.push(env); + reporter.report_environment(&env); found = true; break; } @@ -198,27 +187,4 @@ fn find_in_global_virtual_env_dirs() -> Option { } } } - Some(LocatorResult { - environments, - managers: vec![], - }) -} - -// This is incomplete -// fn find_in_path_env_variable() -> Option { -// let environment = EnvironmentApi::new(); -// PythonOnPath::from(&environment).find() -// } - -fn report_result(result: Option, reporter: &dyn Reporter) { - if let Some(result) = result { - result - .environments - .iter() - .for_each(|e| reporter.report_environment(e)); - result - .managers - .iter() - .for_each(|m| reporter.report_manager(m)); - } } diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index 892f9e2a..3795fed2 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -7,7 +7,6 @@ use pet_core::{ arch::Architecture, python_environment::{PythonEnvironment, PythonEnvironmentCategory}, }; -use pet_utils::env; use regex::Regex; use serde::Deserialize; @@ -30,14 +29,9 @@ fn verify_validity_of_discovered_envs() { let reporter = test::create_reporter(); locators::find_and_report_envs(&reporter); + let result = reporter.get_result(); - let environments = reporter - .reported_environments - .lock() - .unwrap() - .clone() - .into_values() - .collect::>(); + let environments = result.environments; let mut threads = vec![]; for environment in environments { if environment.executable.is_none() { @@ -63,14 +57,8 @@ fn check_if_virtualenvwrapper_exists() { let reporter = test::create_reporter(); locators::find_and_report_envs(&reporter); - - let environments = reporter - .reported_environments - .lock() - .unwrap() - .clone() - .into_values() - .collect::>(); + let result = reporter.get_result(); + let environments = result.environments; assert!( environments.iter().any( @@ -95,14 +83,8 @@ fn check_if_pyenv_virtualenv_exists() { let reporter = test::create_reporter(); locators::find_and_report_envs(&reporter); - - let environments = reporter - .reported_environments - .lock() - .unwrap() - .clone() - .into_values() - .collect::>(); + let result = reporter.get_result(); + let environments = result.environments; assert!( environments.iter().any(