diff --git a/.devcontainer/linux-homebrew/Dockerfile b/.devcontainer/linux-homebrew/Dockerfile new file mode 100644 index 00000000..f79c8ddb --- /dev/null +++ b/.devcontainer/linux-homebrew/Dockerfile @@ -0,0 +1,19 @@ +FROM homebrew/brew + +RUN sh -c "$(curl -fsSL https://github.com/deluan/zsh-in-docker/releases/download/v1.1.5/zsh-in-docker.sh)" -- \ + -t powerlevel10k/powerlevel10k \ + -p git \ + -p git-extras \ + -p https://github.com/zsh-users/zsh-completions +RUN git clone https://github.com/romkatv/powerlevel10k $HOME/.oh-my-zsh/custom/themes/powerlevel10k +RUN curl https://raw.githubusercontent.com/DonJayamanne/vscode-jupyter/containerChanges/.devcontainer/.p10k.zsh > ~/.p10k.zsh +RUN echo "# To customize prompt, run `p10k configure` or edit ~/.p10k.zsh." >> ~/.zshrc +RUN echo "[[ ! -f ~/.p10k.zsh ]] || source ~/.p10k.zsh" >> ~/.zshrc + +# Install Pythone +RUN brew install python@3.12 python@3.11 + +# Install Rust +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc +ENV PATH="/root/.cargo/bin:${PATH}" diff --git a/.devcontainer/linux-homebrew/devcontainer.json b/.devcontainer/linux-homebrew/devcontainer.json new file mode 100644 index 00000000..c9df080a --- /dev/null +++ b/.devcontainer/linux-homebrew/devcontainer.json @@ -0,0 +1,18 @@ +{ + "name": "linux-homebrew", + "build": { + "context": "../..", + "dockerfile": "./Dockerfile" + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python@prerelease", + "esbenp.prettier-vscode", + "rust-lang.rust-analyzer", + "EditorConfig.EditorConfig" + ] + } + }, + "workspaceFolder": "/workspaces/python-environment-tools" +} \ No newline at end of file diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 8c703769..0c9d2951 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -2,8 +2,13 @@ name: PR/CI Check on: pull_request: + branches: + - main + - release* + - release/* + - release-* push: - branches-ignore: + branches: - main - release* - release/* @@ -11,6 +16,9 @@ on: jobs: tests: + # Very generic tests, we don't verify whether the envs are discovered correctly or not. + # However we do ensure that envs that are discovered are valid. + # See other jobs for specific tests. name: Tests runs-on: ${{ matrix.os }} strategy: @@ -26,20 +34,12 @@ jobs: - os: ubuntu-latest target: x86_64-unknown-linux-musl run_cli: "yes" - # - os: ubuntu-latest - # target: aarch64-unknown-linux-gnu - # - os: ubuntu-latest - # target: arm-unknown-linux-gnueabihf - os: macos-latest target: x86_64-apple-darwin run_cli: "yes" - os: macos-14 target: aarch64-apple-darwin run_cli: "yes" - # - os: ubuntu-latest - # target: x86_64-unknown-linux-gnu - # - os: ubuntu-latest - # target: aarch64-unknown-linux-musl steps: - name: Checkout uses: actions/checkout@v4 @@ -151,6 +151,22 @@ jobs: pyenv virtualenv 3.12 pyenv-virtualenv-env1 shell: bash + # region venv + - name: Create .venv + # if: startsWith( matrix.os, 'ubuntu') || startsWith( matrix.os, 'macos') + run: | + python -m venv .venv + shell: bash + + - name: Create .venv2 + # if: startsWith( matrix.os, 'ubuntu') || startsWith( matrix.os, 'macos') + run: | + python -m venv .venv2 + shell: bash + + # endregion venv + + # Rust - name: Rust Tool Chain setup uses: dtolnay/rust-toolchain@stable with: @@ -167,7 +183,194 @@ jobs: shell: bash - name: Run Tests - run: cargo test --frozen --all-features -- --nocapture + # Run integration tests in a single thread, + # We end up creating conda envs and running multiple tests in parallel + # that creat conda envs simultaneously causes issues (sometimes the conda envs do not seem to get created) + # Similar issues were identified in vscode-jupyter tests as well (something to do with conda lock files or the like) + run: cargo test --frozen --features ci -- --nocapture --test-threads=1 + env: + RUST_BACKTRACE: 1 + RUST_LOG: trace + shell: bash + + isolated-tests: + # Some of these tests are very specific and need to be run in isolation. + # E.g. we need to ensure we have a poetry project setup correctly (without .venv created using `pip -m venv .venv`). + # We can try to use the previous `tests` job, but that gets very complicated. + name: Other Tests + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - feature: ci-poetry-global + os: ubuntu-latest + target: x86_64-unknown-linux-musl + - feature: ci-poetry-project + os: ubuntu-latest + target: x86_64-unknown-linux-musl + - feature: ci-poetry-custom + os: ubuntu-latest + target: x86_64-unknown-linux-musl + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Setup Poetry + - name: Set Python 3.x to PATH + if: startsWith( matrix.feature, 'ci-poetry') + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Set Python 3.12 to PATH + if: startsWith( matrix.feature, 'ci-poetry') + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set Python 3.11 to PATH + if: startsWith( matrix.feature, 'ci-poetry') + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install Poetry (envs globally) + if: startsWith( matrix.feature, 'ci-poetry-global') + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b + with: + virtualenvs-create: true + virtualenvs-in-project: false + installer-parallel: true + + - name: Install Poetry (env locally) + if: startsWith( matrix.feature, 'ci-poetry-project') + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install Poetry (env locally) + if: startsWith( matrix.feature, 'ci-poetry-custom') + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b + with: + virtualenvs-create: true + virtualenvs-in-project: false + virtualenvs-path: ~/my-custom-path + installer-parallel: true + + - name: Petry config + if: startsWith( matrix.feature, 'ci-poetry') + run: poetry config --list + shell: bash + + - name: Petry setup + if: startsWith( matrix.feature, 'ci-poetry') + # We want to have 2 envs for this poetry project 3.12 and 3.11. + run: poetry init --name=pet-test --python=^3.11 -q -n + shell: bash + + - name: Petry virtual env setup 3.12 + if: startsWith( matrix.feature, 'ci-poetry') + run: poetry env use python3.12 + shell: bash + + - name: Petry virtual env setup 3.11 + if: startsWith( matrix.feature, 'ci-poetry') + run: poetry env use python3.11 + shell: bash + + - name: Petry list envs + if: startsWith( matrix.feature, 'ci-poetry') + run: poetry env list + shell: bash + + - name: Petry pyproject.toml + if: startsWith( matrix.feature, 'ci-poetry') + run: cat pyproject.toml + shell: bash + + # # Dump env vars + # - name: Env + # run: set + # shell: bash + + # Rust + - name: Rust Tool Chain setup + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + targets: ${{ matrix.target }} + + - name: Cargo Fetch + run: cargo fetch + shell: bash + + - name: Find Environments + run: cargo run --release --target ${{ matrix.target }} + shell: bash + + - name: Run Tests + # Run integration tests in a single thread, + # We end up creating conda envs and running multiple tests in parallel + # that creat conda envs simultaneously causes issues (sometimes the conda envs do not seem to get created) + # Similar issues were identified in vscode-jupyter tests as well (something to do with conda lock files or the like) + run: cargo test --frozen --features ${{ matrix.feature }} -- --nocapture --test-threads=1 + env: + RUST_BACKTRACE: 1 + RUST_LOG: trace + shell: bash + + container-tests: + # These tests are required as its not easy/possible to use the previous jobs. + # E.g. we need to test against the jupyter container, as we found some issues specific to that env. + name: Tests in Containers + container: + image: ${{ matrix.image }} + options: --user=root + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - feature: ci-jupyter-container + os: ubuntu-latest + # For Tests again the container used in https://github.com/github/codespaces-jupyter + image: mcr.microsoft.com/devcontainers/universal:2.11.1 + target: x86_64-unknown-linux-musl + - feature: ci-homebrew-container + os: ubuntu-latest + # For Homebrew in Ubuntu + image: homebrew/brew + target: x86_64-unknown-linux-musl + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Homebrew + - name: Homebrew Python + if: startsWith( matrix.image, 'homebrew') + run: brew install python@3.12 python@3.11 + shell: bash + + # Rust + - name: Rust Tool Chain setup + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + targets: ${{ matrix.target }} + + - name: Cargo Fetch + run: cargo fetch + shell: bash + + - name: Find Environments + run: cargo run --release --target ${{ matrix.target }} -- find -v + shell: bash + + - name: Run Tests + run: cargo test --frozen --features ${{ matrix.feature }} -- --nocapture shell: bash builds: diff --git a/Cargo.lock b/Cargo.lock index 232ff92d..5b649744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -333,6 +333,7 @@ dependencies = [ name = "pet-conda" version = "0.1.0" dependencies = [ + "env_logger", "lazy_static", "log", "msvc_spectre_libs", diff --git a/crates/pet-conda/Cargo.toml b/crates/pet-conda/Cargo.toml index e3c91279..999568dd 100644 --- a/crates/pet-conda/Cargo.toml +++ b/crates/pet-conda/Cargo.toml @@ -16,7 +16,7 @@ pet-core = { path = "../pet-core" } log = "0.4.21" regex = "1.10.4" pet-reporter = { path = "../pet-reporter" } - +env_logger = "0.10.2" [features] ci = [] \ No newline at end of file diff --git a/crates/pet-conda/src/environment_locations.rs b/crates/pet-conda/src/environment_locations.rs index 21c2072a..55327235 100644 --- a/crates/pet-conda/src/environment_locations.rs +++ b/crates/pet-conda/src/environment_locations.rs @@ -65,8 +65,10 @@ pub fn get_conda_environment_paths( */ fn get_conda_environment_paths_from_conda_rc(env_vars: &EnvVariables) -> Vec { if let Some(conda_rc) = Condarc::from(env_vars) { + trace!("Conda environments in .condarc {:?}", conda_rc.env_dirs); conda_rc.env_dirs } else { + trace!("No Conda environments in .condarc"); vec![] } } @@ -98,6 +100,7 @@ fn get_conda_environment_paths_from_known_paths(env_vars: &EnvVariables) -> Vec< } } env_paths.append(&mut env_vars.known_global_search_locations.clone()); + trace!("Conda environments in known paths {:?}", env_paths); env_paths } @@ -118,6 +121,7 @@ fn get_conda_environment_paths_from_additional_paths( } } env_paths.append(&mut additional_env_dirs.clone()); + trace!("Conda environments in additional paths {:?}", env_paths); env_paths } @@ -170,7 +174,9 @@ pub fn get_conda_envs_from_environment_txt(env_vars: &EnvVariables) -> Vec Option { +fn get_python_in_bin(env: &PythonEnv, is_64bit: bool) -> Option { // If we do not have the prefix, then do not try // This method will be called with resolved Python where prefix & version is available. if env.version.clone().is_none() || env.prefix.clone().is_none() { @@ -211,6 +212,11 @@ fn get_python_in_bin(env: &PythonEnv) -> Option { PythonEnvironmentBuilder::new(PythonEnvironmentCategory::LinuxGlobal) .executable(Some(executable)) .version(env.version.clone()) + .arch(if is_64bit { + Some(Architecture::X64) + } else { + Some(Architecture::X86) + }) .prefix(env.prefix.clone()) .symlinks(Some(symlinks)) .build(), diff --git a/crates/pet-poetry/src/config.rs b/crates/pet-poetry/src/config.rs index ec03c6f0..67503c0d 100644 --- a/crates/pet-poetry/src/config.rs +++ b/crates/pet-poetry/src/config.rs @@ -27,9 +27,10 @@ impl Config { virtualenvs_in_project: Option, ) -> Self { trace!( - "Poetry config file: {:?} with virtualenv.path {:?}", + "Poetry config file => {:?}, virtualenv.path => {:?}, virtualenvs_in_project => {:?}", file, - virtualenvs_path + virtualenvs_path, + virtualenvs_in_project ); Config { file, @@ -54,6 +55,26 @@ impl Config { fn create_config(file: Option, env: &EnvVariables) -> Option { let cfg = file.clone().and_then(|f| parse(&f)); if let Some(virtualenvs_path) = &cfg.clone().and_then(|cfg| cfg.virtualenvs_path) { + let mut virtualenvs_path = virtualenvs_path.clone(); + trace!("Poetry virtualenvs path => {:?}", virtualenvs_path); + if virtualenvs_path + .to_string_lossy() + .to_lowercase() + .contains("{cache-dir}") + { + if let Some(cache_dir) = &get_default_cache_dir(env) { + virtualenvs_path = PathBuf::from( + virtualenvs_path + .to_string_lossy() + .replace("{cache-dir}", cache_dir.to_string_lossy().as_ref()), + ); + trace!( + "Poetry virtualenvs path after replacing cache-dir => {:?}", + virtualenvs_path + ); + } + } + return Some(Config::new( file.clone(), virtualenvs_path.clone(), diff --git a/crates/pet-poetry/src/environment_locations.rs b/crates/pet-poetry/src/environment_locations.rs index 4bcd3be0..dd701045 100644 --- a/crates/pet-poetry/src/environment_locations.rs +++ b/crates/pet-poetry/src/environment_locations.rs @@ -74,7 +74,8 @@ pub fn list_environments( .unwrap_or_default() .to_str() .unwrap_or_default(); - if name.starts_with(&virtualenv_prefix) { + // Look for .venv as well, in case we create the virtual envs in the local project folder. + if name.starts_with(&virtualenv_prefix) || name.starts_with(".venv") { if let Some(env) = create_poetry_env(&virtual_env, project_dir.clone(), None) { envs.push(env); } @@ -91,6 +92,7 @@ fn list_all_environments_from_project_config( env: &EnvVariables, ) -> Option> { let local = Config::find_local(path, env); + trace!("Poetry Project ({:?}) config file => {:?}", path, local); let mut envs = vec![]; if let Some(local) = &local { @@ -121,19 +123,32 @@ fn should_use_local_venv_as_poetry_env( if let Some(poetry_virtualenvs_in_project) = local.clone().and_then(|c| c.virtualenvs_in_project) { + trace!( + "Poetry virtualenvs_in_project from local config file: {}", + poetry_virtualenvs_in_project + ); return poetry_virtualenvs_in_project; } // Given preference to env variable. if let Some(poetry_virtualenvs_in_project) = env.poetry_virtualenvs_in_project { + trace!( + "Poetry virtualenvs_in_project from Env Variable: {}", + poetry_virtualenvs_in_project + ); return poetry_virtualenvs_in_project; } // Check global config setting. - global + let value = global .clone() .and_then(|config| config.virtualenvs_in_project) - .unwrap_or_default() + .unwrap_or_default(); + trace!( + "Poetry virtualenvs_in_project from global config file: {}", + value + ); + value } fn list_all_environments_from_config(cfg: &Config) -> Option> { diff --git a/crates/pet-poetry/tests/config_test.rs b/crates/pet-poetry/tests/config_test.rs index 377ef076..19d100d2 100644 --- a/crates/pet-poetry/tests/config_test.rs +++ b/crates/pet-poetry/tests/config_test.rs @@ -4,7 +4,8 @@ mod common; #[cfg(unix)] -#[test] +#[cfg_attr(any(feature = "ci",), test)] +#[allow(dead_code)] fn global_config_with_defaults() { use common::create_env_variables; use common::resolve_test_path; @@ -36,7 +37,8 @@ fn global_config_with_defaults() { } #[cfg(unix)] -#[test] +#[cfg_attr(any(feature = "ci",), test)] +#[allow(dead_code)] fn global_config_with_specific_values() { use std::path::PathBuf; @@ -82,7 +84,9 @@ fn global_config_with_specific_values() { } #[cfg(unix)] -#[test] +#[cfg_attr(any(feature = "ci",), test)] +#[allow(dead_code)] + fn local_config_with_specific_values() { use std::path::PathBuf; diff --git a/crates/pet-python-utils/src/executable.rs b/crates/pet-python-utils/src/executable.rs index eae51411..2cecf783 100644 --- a/crates/pet-python-utils/src/executable.rs +++ b/crates/pet-python-utils/src/executable.rs @@ -89,6 +89,8 @@ pub fn find_executables>(env_path: T) -> Vec { } } + // Ensure the exe `python` is first, instead of `python3.10` + python_executables.sort(); python_executables } diff --git a/crates/pet/Cargo.toml b/crates/pet/Cargo.toml index 10a56569..35ef8b0e 100644 --- a/crates/pet/Cargo.toml +++ b/crates/pet/Cargo.toml @@ -44,3 +44,8 @@ lazy_static = "1.4.0" [features] ci = [] +ci-jupyter-container = [] +ci-homebrew-container = [] +ci-poetry-global = [] +ci-poetry-project = [] +ci-poetry-custom = [] diff --git a/crates/pet/tests/ci_homebrew_container.rs b/crates/pet/tests/ci_homebrew_container.rs new file mode 100644 index 00000000..1fc12ddd --- /dev/null +++ b/crates/pet/tests/ci_homebrew_container.rs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[cfg(unix)] +#[cfg_attr(feature = "ci-homebrew-container", test)] +#[allow(dead_code)] +fn verify_python_in_homebrew_contaner() { + use pet::{find::find_and_report_envs, locators::create_locators}; + use pet_conda::Conda; + use pet_core::{ + os_environment::EnvironmentApi, + python_environment::{PythonEnvironment, PythonEnvironmentCategory}, + }; + use pet_reporter::test; + use std::{path::PathBuf, sync::Arc}; + + let reporter = test::create_reporter(); + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + + find_and_report_envs( + &reporter, + Default::default(), + &create_locators(conda_locator.clone()), + conda_locator, + ); + let result = reporter.get_result(); + + let environments = result.environments; + + let python3_12 = PythonEnvironment { + category: PythonEnvironmentCategory::Homebrew, + executable: Some(PathBuf::from("/home/linuxbrew/.linuxbrew/bin/python3")), + version: Some("3.12.4".to_string()), // This can change on CI, so we don't check it + symlinks: Some(vec![ + PathBuf::from("/home/linuxbrew/.linuxbrew/bin/python3"), + PathBuf::from("/home/linuxbrew/.linuxbrew/bin/python3.12"), + // On CI the Python version can change with minor updates, so we don't check the full version. + // PathBuf::from("/home/linuxbrew/.linuxbrew/Cellar/python@3.12/3.12.4/bin/python3.12"), + ]), + ..Default::default() + }; + let python3_11 = PythonEnvironment { + category: PythonEnvironmentCategory::Homebrew, + executable: Some(PathBuf::from("/home/linuxbrew/.linuxbrew/bin/python3.11")), + version: Some("3.11.9".to_string()), // This can change on CI, so we don't check it + symlinks: Some(vec![ + PathBuf::from("/home/linuxbrew/.linuxbrew/bin/python3.11"), + // On CI the Python version can change with minor updates, so we don't check the full version. + // PathBuf::from("/home/linuxbrew/.linuxbrew/Cellar/python@3.11/3.11.9/bin/python3.11"), + ]), + ..Default::default() + }; + + assert_eq!(environments.len(), 2); + + for env in [python3_11, python3_12].iter() { + let python_env = environments + .iter() + .find(|e| e.executable == env.executable) + .expect(format!("Expected to find python environment {:?}", env.executable).as_str()); + assert_eq!(python_env.executable, env.executable); + assert_eq!(python_env.category, env.category); + assert_eq!(python_env.manager, env.manager); + // Compare the first 4 parts (3.12) + assert_eq!( + python_env.version.clone().unwrap_or_default()[..4], + env.version.clone().unwrap_or_default()[..4] + ); + + // We know the symlinks contain the full version, hence exclude the paths that contain the full version. + let python_env_symlinks = python_env + .symlinks + .clone() + .unwrap_or_default() + .into_iter() + .filter(|p| { + !p.to_string_lossy() + .contains(&env.version.clone().unwrap_or_default()) + }) + .collect::>(); + assert_eq!( + python_env_symlinks, + env.symlinks.clone().unwrap_or_default() + ); + } +} diff --git a/crates/pet/tests/ci_jupyter_container.rs b/crates/pet/tests/ci_jupyter_container.rs new file mode 100644 index 00000000..adacd2c6 --- /dev/null +++ b/crates/pet/tests/ci_jupyter_container.rs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[cfg(unix)] +#[cfg_attr(feature = "ci-jupyter-container", test)] +#[allow(dead_code)] +/// Tests again the container used in https://github.com/github/codespaces-jupyter +fn verify_python_in_jupyter_contaner() { + use pet::{find::find_and_report_envs, locators::create_locators}; + use pet_conda::Conda; + use pet_core::{ + arch::Architecture, + manager::{EnvManager, EnvManagerType}, + os_environment::EnvironmentApi, + python_environment::{PythonEnvironment, PythonEnvironmentCategory}, + }; + use pet_reporter::test; + use std::{path::PathBuf, sync::Arc}; + + let reporter = test::create_reporter(); + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + + find_and_report_envs( + &reporter, + Default::default(), + &create_locators(conda_locator.clone()), + conda_locator, + ); + let result = reporter.get_result(); + + let environments = result.environments; + + let conda = PythonEnvironment { + category: PythonEnvironmentCategory::Conda, + name: Some("base".to_string()), + executable: Some(PathBuf::from("/opt/conda/bin/python")), + prefix: Some(PathBuf::from("/opt/conda")), + version: Some("3.12.3".to_string()), + arch: Some(Architecture::X64), + symlinks: Some(vec![ + PathBuf::from("/opt/conda/bin/python"), + PathBuf::from("/opt/conda/bin/python3"), + PathBuf::from("/opt/conda/bin/python3.1"), + PathBuf::from("/opt/conda/bin/python3.12"), + ]), + manager: Some(EnvManager { + tool: EnvManagerType::Conda, + executable: PathBuf::from("/opt/conda/bin/conda"), + version: Some("24.5.0".to_string()), + }), + ..Default::default() + }; + let codespace_python = PythonEnvironment { + category: PythonEnvironmentCategory::GlobalPaths, + executable: Some(PathBuf::from("/home/codespace/.python/current/bin/python")), + prefix: Some(PathBuf::from("/usr/local/python/3.10.13")), + version: Some("3.10.13.final.0".to_string()), + arch: Some(Architecture::X64), + symlinks: Some(vec![ + PathBuf::from("/home/codespace/.python/current/bin/python"), + PathBuf::from("/home/codespace/.python/current/bin/python3"), + PathBuf::from("/home/codespace/.python/current/bin/python3.10"), + ]), + manager: None, + ..Default::default() + }; + let current_python = PythonEnvironment { + category: PythonEnvironmentCategory::GlobalPaths, + executable: Some(PathBuf::from("/usr/local/python/current/bin/python")), + prefix: Some(PathBuf::from("/usr/local/python/3.10.13")), + version: Some("3.10.13.final.0".to_string()), + arch: Some(Architecture::X64), + symlinks: Some(vec![ + PathBuf::from("/usr/local/python/current/bin/python"), + PathBuf::from("/usr/local/python/current/bin/python3"), + PathBuf::from("/usr/local/python/current/bin/python3.10"), + ]), + manager: None, + ..Default::default() + }; + let usr_bin_python = PythonEnvironment { + category: PythonEnvironmentCategory::LinuxGlobal, + executable: Some(PathBuf::from("/usr/bin/python3")), + prefix: Some(PathBuf::from("/usr")), + version: Some("3.8.10.final.0".to_string()), + arch: Some(Architecture::X64), + symlinks: Some(vec![ + PathBuf::from("/usr/bin/python3"), + PathBuf::from("/usr/bin/python3.8"), + ]), + manager: None, + ..Default::default() + }; + let bin_python = PythonEnvironment { + category: PythonEnvironmentCategory::LinuxGlobal, + executable: Some(PathBuf::from("/bin/python3")), + prefix: Some(PathBuf::from("/usr")), + version: Some("3.8.10.final.0".to_string()), + arch: Some(Architecture::X64), + symlinks: Some(vec![ + PathBuf::from("/bin/python3"), + PathBuf::from("/bin/python3.8"), + ]), + manager: None, + ..Default::default() + }; + + for env in [ + conda, + codespace_python, + current_python, + usr_bin_python, + bin_python, + ] + .iter() + { + let python_env = environments + .iter() + .find(|e| e.executable == env.executable) + .expect(format!("Expected to find python environment {:?}", env.executable).as_str()); + assert_eq!(python_env.executable, env.executable); + assert_eq!(python_env.category, env.category); + assert_eq!(python_env.symlinks, env.symlinks); + assert_eq!(python_env.manager, env.manager); + assert_eq!(python_env.name, env.name); + assert_eq!(python_env.version, env.version); + assert_eq!(python_env.arch, env.arch); + + // known issue https://github.com/microsoft/python-environment-tools/issues/64 + if env.executable == Some(PathBuf::from("/home/codespace/.python/current/bin/python")) { + assert!( + python_env.prefix == Some(PathBuf::from("/home/codespace/.python/current")) + || python_env.prefix == Some(PathBuf::from("/usr/local/python/3.10.13")), + "Expected {:?} to be {:?} or {:?}", + python_env.prefix, + "/home/codespace/.python/current", + "/usr/local/python/3.10.13" + ); + } + } +} diff --git a/crates/pet/tests/ci_poetry.rs b/crates/pet/tests/ci_poetry.rs new file mode 100644 index 00000000..ecbde77d --- /dev/null +++ b/crates/pet/tests/ci_poetry.rs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[cfg(unix)] +#[cfg_attr(any(feature = "ci-poetry-global", feature = "ci-poetry-custom"), test)] +#[allow(dead_code)] +/// This is a test with Poetry for current directory with Python 3.12 and 3.11 and envs are created in regular global cache directory +fn verify_ci_poetry_global() { + use pet::{find::find_and_report_envs, locators::create_locators}; + use pet_conda::Conda; + use pet_core::{ + manager::EnvManagerType, + os_environment::EnvironmentApi, + python_environment::{PythonEnvironment, PythonEnvironmentCategory}, + Configuration, + }; + use pet_reporter::test; + use std::{env, path::PathBuf, sync::Arc}; + + let project_dir = PathBuf::from(env::var("GITHUB_WORKSPACE").unwrap_or_default()); + let reporter = test::create_reporter(); + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + let mut config = Configuration::default(); + config.search_paths = Some(vec![project_dir.clone()]); + let locators = create_locators(conda_locator.clone()); + for locator in locators.iter() { + locator.configure(&config); + } + + find_and_report_envs(&reporter, Default::default(), &locators, conda_locator); + + let result = reporter.get_result(); + + let environments = result.environments; + + result + .managers + .iter() + .find(|m| m.tool == EnvManagerType::Poetry) + .expect("Poetry manager not found"); + + let poetry_envs = environments + .iter() + .filter(|e| { + e.category == PythonEnvironmentCategory::Poetry + && e.project == Some(project_dir.clone()) + }) + .collect::>(); + + assert_eq!(poetry_envs.len(), 2); + + poetry_envs + .iter() + .find(|e| e.version.clone().unwrap_or_default().starts_with("3.12")) + .expect("Python 3.12 not found"); + poetry_envs + .iter() + .find(|e| e.version.clone().unwrap_or_default().starts_with("3.11")) + .expect("Python 3.12 not found"); +} + +#[cfg(unix)] +#[cfg_attr(feature = "ci-poetry-project", test)] +#[allow(dead_code)] +/// This is a test with Poetry for current directory with Python 3.11 and created as .venv in project directory. +fn verify_ci_poetry_project() { + use pet::{find::find_and_report_envs, locators::create_locators}; + use pet_conda::Conda; + use pet_core::{ + manager::EnvManagerType, + os_environment::EnvironmentApi, + python_environment::{PythonEnvironment, PythonEnvironmentCategory}, + Configuration, + }; + use pet_reporter::test; + use std::{env, path::PathBuf, sync::Arc}; + + let project_dir = PathBuf::from(env::var("GITHUB_WORKSPACE").unwrap_or_default()); + let reporter = test::create_reporter(); + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + let mut config = Configuration::default(); + config.search_paths = Some(vec![project_dir.clone()]); + let locators = create_locators(conda_locator.clone()); + for locator in locators.iter() { + locator.configure(&config); + } + + find_and_report_envs(&reporter, Default::default(), &locators, conda_locator); + + let result = reporter.get_result(); + + let environments = result.environments; + + result + .managers + .iter() + .find(|m| m.tool == EnvManagerType::Poetry) + .expect("Poetry manager not found"); + + let poetry_envs = environments + .iter() + .filter(|e| { + e.category == PythonEnvironmentCategory::Poetry + && e.project == Some(project_dir.clone()) + }) + .collect::>(); + + assert_eq!(poetry_envs.len(), 1); + + assert!( + poetry_envs[0] + .version + .clone() + .unwrap_or_default() + .starts_with("3.11"), + "Python 3.11 not found" + ); + assert_eq!( + poetry_envs[0].prefix.clone().unwrap_or_default(), + project_dir.join(".venv") + ); +} diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index a145725a..14a51e45 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; -use common::resolve_test_path; +use common::{does_version_match, resolve_test_path}; use lazy_static::lazy_static; use pet_core::{ arch::Architecture, @@ -20,26 +20,36 @@ lazy_static! { mod common; #[cfg(unix)] -#[cfg_attr(feature = "ci", test)] +#[cfg_attr( + any( + feature = "ci", + feature = "ci-jupyter-container", + feature = "ci-homebrew-container" + ), + test +)] #[allow(dead_code)] // We should detect the conda install along with the base env fn verify_validity_of_discovered_envs() { use pet::{find::find_and_report_envs, locators::create_locators}; use pet_conda::Conda; - use pet_core::os_environment::EnvironmentApi; + use pet_core::{os_environment::EnvironmentApi, Configuration}; use pet_reporter::test; - use std::{sync::Arc, thread}; + use std::{env, sync::Arc, thread}; let reporter = test::create_reporter(); let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); + let mut config = Configuration::default(); + if let Ok(cwd) = env::current_dir() { + config.search_paths = Some(vec![cwd]); + } + let locators = create_locators(conda_locator.clone()); + for locator in locators.iter() { + locator.configure(&config); + } - find_and_report_envs( - &reporter, - Default::default(), - &create_locators(conda_locator.clone()), - conda_locator, - ); + find_and_report_envs(&reporter, Default::default(), &locators, conda_locator); let result = reporter.get_result(); let environments = result.environments; @@ -175,12 +185,29 @@ fn verify_validity_of_interpreter_info(environment: PythonEnvironment) { } } if let Some(prefix) = environment.clone().prefix { - assert_eq!( - prefix.to_str().unwrap(), - interpreter_info.clone().sys_prefix, - "Prefix mismatch for {:?}", - environment.clone() - ); + if interpreter_info.clone().executable == "/usr/local/python/current/bin/python" + && (prefix.to_str().unwrap() == "/usr/local/python/current" + && interpreter_info.clone().sys_prefix == "/usr/local/python/3.10.13") + || (prefix.to_str().unwrap() == "/usr/local/python/3.10.13" + && interpreter_info.clone().sys_prefix == "/usr/local/python/current") + { + // known issue https://github.com/microsoft/python-environment-tools/issues/64 + } else if interpreter_info.clone().executable + == "/home/codespace/.python/current/bin/python" + && (prefix.to_str().unwrap() == "/home/codespace/.python/current" + && interpreter_info.clone().sys_prefix == "/usr/local/python/3.10.13") + || (prefix.to_str().unwrap() == "/usr/local/python/3.10.13" + && interpreter_info.clone().sys_prefix == "/home/codespace/.python/current") + { + // known issue https://github.com/microsoft/python-environment-tools/issues/64 + } else { + assert_eq!( + prefix.to_str().unwrap(), + interpreter_info.clone().sys_prefix, + "Prefix mismatch for {:?}", + environment.clone() + ); + } } if let Some(arch) = environment.clone().arch { let expected_arch = if interpreter_info.clone().is64_bit { @@ -197,9 +224,8 @@ fn verify_validity_of_interpreter_info(environment: PythonEnvironment) { } if let Some(version) = environment.clone().version { let expected_version = &interpreter_info.clone().sys_version; - let version = get_version(&version); assert!( - expected_version.starts_with(&version), + does_version_match(&version, expected_version), "Version mismatch for (expected {:?} to start with {:?}) for {:?}", expected_version, version, @@ -345,14 +371,3 @@ fn get_python_interpreter_info(cli: &Vec) -> InterpreterInfo { let info: InterpreterInfo = serde_json::from_str(&output).unwrap(); info } - -fn get_version(value: &String) -> String { - // Regex to extract just the d.d.d version from the full version string - let captures = PYTHON_VERSION.captures(value).unwrap(); - let version = captures.get(1).unwrap().as_str().to_string(); - if version.ends_with('.') { - version[..version.len() - 1].to_string() - } else { - version - } -} diff --git a/crates/pet/tests/common.rs b/crates/pet/tests/common.rs index 2267530c..634fc59d 100644 --- a/crates/pet/tests/common.rs +++ b/crates/pet/tests/common.rs @@ -1,8 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use lazy_static::lazy_static; +use regex::Regex; use std::path::PathBuf; +lazy_static! { + static ref PYTHON_VERSION: Regex = Regex::new("([\\d+\\.?]*).*") + .expect("error parsing Version regex for Python Version in test"); + static ref PYTHON_FULLVERSION: Regex = Regex::new("(\\d+\\.?\\d+\\.?\\d+).*") + .expect("error parsing Version regex for Python Version in test"); +} + #[allow(dead_code)] pub fn resolve_test_path(paths: &[&str]) -> PathBuf { let mut root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests"); @@ -11,3 +20,25 @@ pub fn resolve_test_path(paths: &[&str]) -> PathBuf { root } + +#[allow(dead_code)] +pub fn does_version_match(version: &String, expected_version: &String) -> bool { + let version = get_version(version); + expected_version.starts_with(&version) +} + +fn get_version(value: &String) -> String { + // Regex to extract just the d.d.d version from the full version string + let captures = PYTHON_VERSION.captures(value).unwrap(); + let version = captures.get(1).unwrap().as_str().to_string(); + if version.ends_with('.') { + version[..version.len() - 1].to_string() + } else { + version + } +} + +#[allow(dead_code)] +pub fn is_valid_version(value: &String) -> bool { + PYTHON_FULLVERSION.is_match(value) +}