From fd6e392a091de327a3a795caf50f86b81570ede9 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 14:58:19 +1000 Subject: [PATCH 1/8] Update samples --- .gitignore | 5 +- crates/pet-core/src/lib.rs | 6 +- crates/pet-poetry/src/lib.rs | 18 ++--- crates/pet/src/find.rs | 12 +-- crates/pet/src/jsonrpc.rs | 17 +++- crates/pet/src/lib.rs | 2 +- crates/pet/tests/ci_poetry.rs | 4 +- crates/pet/tests/ci_test.rs | 6 +- docs/sample.js | 148 ++++++++++++++++++++++++++++++++++ 9 files changed, 189 insertions(+), 29 deletions(-) create mode 100644 docs/sample.js diff --git a/.gitignore b/.gitignore index f418d35d..f3f71df4 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,7 @@ target/ # Testing directories .venv/ tmp/ -temp/ \ No newline at end of file +temp/ +docs/node_modules/ +docs/package.json +docs/package-lock.json \ No newline at end of file diff --git a/crates/pet-core/src/lib.rs b/crates/pet-core/src/lib.rs index 3c3209ce..6d90d32f 100644 --- a/crates/pet-core/src/lib.rs +++ b/crates/pet-core/src/lib.rs @@ -24,13 +24,13 @@ pub struct LocatorResult { #[derive(Debug, Default, Clone)] pub struct Configuration { /// These are paths like workspace folders, where we can look for environments. - pub search_paths: Option>, + pub project_directories: Option>, pub conda_executable: Option, pub poetry_executable: Option, /// Custom locations where environments can be found. /// These are different from search_paths, as these are specific directories where environments are expected. - /// search_paths on the other hand can be any directory such as a workspace folder, where envs might never exist. - pub environment_paths: Option>, + /// environment_directories on the other hand can be any directory such as a workspace folder, where envs might never exist. + pub environment_directories: Option>, } pub trait Locator: Send + Sync { diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index 8a9d9ac2..f31566e5 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -29,7 +29,7 @@ pub mod manager; mod pyproject_toml; pub struct Poetry { - pub project_dirs: Arc>>, + pub project_directories: Arc>>, pub env_vars: EnvVariables, pub poetry_executable: Arc>>, searched: AtomicBool, @@ -41,7 +41,7 @@ impl Poetry { Poetry { searched: AtomicBool::new(false), search_result: Arc::new(Mutex::new(None)), - project_dirs: Arc::new(Mutex::new(vec![])), + project_directories: Arc::new(Mutex::new(vec![])), env_vars: EnvVariables::from(environment), poetry_executable: Arc::new(Mutex::new(None)), } @@ -57,7 +57,7 @@ impl Poetry { let environments_using_spawn = environment_locations_spawn::list_environments( &manager.executable, - self.project_dirs.lock().unwrap().clone(), + self.project_directories.lock().unwrap().clone(), &manager, ) .iter() @@ -108,7 +108,7 @@ impl Poetry { if let Some(manager) = &manager { result.managers.push(manager.to_manager()); } - if let Ok(values) = self.project_dirs.lock() { + if let Ok(values) = self.project_directories.lock() { let project_dirs = values.clone(); drop(values); let envs = list_environments(&self.env_vars, &project_dirs.clone(), manager) @@ -139,13 +139,13 @@ impl Locator for Poetry { "Poetry" } fn configure(&self, config: &Configuration) { - if let Some(search_paths) = &config.search_paths { - self.project_dirs.lock().unwrap().clear(); - if !search_paths.is_empty() { - self.project_dirs + if let Some(project_directories) = &config.project_directories { + self.project_directories.lock().unwrap().clear(); + if !project_directories.is_empty() { + self.project_directories .lock() .unwrap() - .extend(search_paths.clone()); + .extend(project_directories.clone()); } } if let Some(exe) = &config.poetry_executable { diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index bd0baf02..8df7f72a 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -49,8 +49,8 @@ pub fn find_and_report_envs( let start = std::time::Instant::now(); // From settings - let environment_paths = configuration.environment_paths.unwrap_or_default(); - let search_paths = configuration.search_paths.unwrap_or_default(); + let environment_directories = configuration.environment_directories.unwrap_or_default(); + let project_directories = configuration.project_directories.unwrap_or_default(); let conda_executable = configuration.conda_executable; thread::scope(|s| { // 1. Find using known global locators. @@ -123,7 +123,7 @@ pub fn find_and_report_envs( environment.get_env_var("XDG_DATA_HOME".into()), environment.get_user_home(), ), - environment_paths, + environment_directories, ] .concat(); let global_env_search_paths: Vec = @@ -152,16 +152,16 @@ pub fn find_and_report_envs( // & users can have a lot of workspace folders and can have a large number fo files/directories // that could the discovery. s.spawn(|| { - if search_paths.is_empty() { + if project_directories.is_empty() { return; } trace!( "Searching for environments in custom folders: {:?}", - search_paths + project_directories ); let start = std::time::Instant::now(); find_python_environments_in_workspace_folders_recursive( - search_paths, + project_directories, reporter, locators, ); diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index 9d65a1c8..63a6bb20 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -57,13 +57,13 @@ pub fn start_jsonrpc_server() { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RequestOptions { /// These are paths like workspace folders, where we can look for environments. - pub search_paths: Option>, + pub project_directories: Option>, pub conda_executable: Option, pub poetry_executable: Option, /// Custom locations where environments can be found. /// These are different from search_paths, as these are specific directories where environments are expected. /// search_paths on the other hand can be any directory such as a workspace folder, where envs might never exist. - pub environment_paths: Option>, + pub environment_directories: Option>, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -85,8 +85,11 @@ pub fn handle_refresh(context: Arc, id: u32, params: Value) { // Start in a new thread, we can have multiple requests. thread::spawn(move || { let mut cfg = context.configuration.write().unwrap(); - cfg.search_paths = request_options.search_paths; + cfg.project_directories = request_options.project_directories; cfg.conda_executable = request_options.conda_executable; + cfg.environment_directories = request_options.environment_directories; + cfg.poetry_executable = request_options.poetry_executable; + trace!("Start refreshing environments, config: {:?}", cfg); drop(cfg); let config = context.configuration.read().unwrap().clone(); for locator in context.locators.iter() { @@ -115,6 +118,7 @@ pub fn handle_refresh(context: Arc, id: u32, params: Value) { "Environments in custom search paths found in {:?}", summary.find_search_paths_time ); + trace!("Finished refreshing environments in {:?}", summary.time); send_reply(id, Some(RefreshResult::new(summary.time))); }); } @@ -149,7 +153,12 @@ pub fn handle_resolve(context: Arc, id: u32, params: Value) { match serde_json::from_value::(params.clone()) { Ok(request_options) => { let executable = request_options.executable.clone(); - let search_paths = context.configuration.read().unwrap().clone().search_paths; + let search_paths = context + .configuration + .read() + .unwrap() + .clone() + .project_directories; let search_paths = search_paths.unwrap_or_default(); // Start in a new thread, we can have multiple resolve requests. thread::spawn(move || { diff --git a/crates/pet/src/lib.rs b/crates/pet/src/lib.rs index eca72a18..2039d266 100644 --- a/crates/pet/src/lib.rs +++ b/crates/pet/src/lib.rs @@ -27,7 +27,7 @@ pub fn find_and_report_envs_stdio(print_list: bool, print_summary: bool, verbose let mut config = Configuration::default(); if let Ok(cwd) = env::current_dir() { - config.search_paths = Some(vec![cwd]); + config.project_directories = Some(vec![cwd]); } let locators = create_locators(conda_locator.clone()); for locator in locators.iter() { diff --git a/crates/pet/tests/ci_poetry.rs b/crates/pet/tests/ci_poetry.rs index 7a0764c8..b0e6c74f 100644 --- a/crates/pet/tests/ci_poetry.rs +++ b/crates/pet/tests/ci_poetry.rs @@ -23,7 +23,7 @@ fn verify_ci_poetry_global() { let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); let mut config = Configuration::default(); - config.search_paths = Some(vec![project_dir.clone()]); + config.project_directories = Some(vec![project_dir.clone()]); let locators = create_locators(conda_locator.clone()); for locator in locators.iter() { locator.configure(&config); @@ -84,7 +84,7 @@ fn verify_ci_poetry_project() { let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); let mut config = Configuration::default(); - config.search_paths = Some(vec![project_dir.clone()]); + config.project_directories = Some(vec![project_dir.clone()]); let locators = create_locators(conda_locator.clone()); for locator in locators.iter() { locator.configure(&config); diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index f10b7a7c..976d484d 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -71,7 +71,7 @@ fn verify_validity_of_discovered_envs() { let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); let mut config = Configuration::default(); - config.search_paths = Some(vec![project_dir.clone()]); + config.project_directories = Some(vec![project_dir.clone()]); let locators = create_locators(conda_locator.clone()); for locator in locators.iter() { locator.configure(&config); @@ -336,7 +336,7 @@ fn verify_we_can_get_same_env_info_using_from_with_exe( let conda_locator = Arc::new(Conda::from(&os_environment)); let mut config = Configuration::default(); let search_paths = vec![project_dir.clone()]; - config.search_paths = Some(search_paths.clone()); + config.project_directories = Some(search_paths.clone()); let locators = create_locators(conda_locator.clone()); for locator in locators.iter() { locator.configure(&config); @@ -522,7 +522,7 @@ fn verify_we_can_get_same_env_info_using_resolve_with_exe( let os_environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&os_environment)); let mut config = Configuration::default(); - config.search_paths = Some(vec![project_dir.clone()]); + config.project_directories = Some(vec![project_dir.clone()]); let locators = create_locators(conda_locator.clone()); for locator in locators.iter() { locator.configure(&config); diff --git a/docs/sample.js b/docs/sample.js new file mode 100644 index 00000000..1e871719 --- /dev/null +++ b/docs/sample.js @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { + createMessageConnection, + StreamMessageReader, + StreamMessageWriter, +} = require("vscode-jsonrpc/node"); +const { spawn } = require("child_process"); +const { PassThrough } = require("stream"); + +const PET_EXE = "../target/debug/pet"; + +const environments = []; + +async function start() { + const readable = new PassThrough(); + const writable = new PassThrough(); + const proc = spawn(PET_EXE, ["server"], { + env: process.env, + }); + proc.stdout.pipe(readable, { end: false }); + proc.stderr.on("data", (data) => console.error(data.toString())); + writable.pipe(proc.stdin, { end: false }); + const connection = createMessageConnection( + new StreamMessageReader(readable), + new StreamMessageWriter(writable) + ); + connection.onError((ex) => console.error("Connection Error:", ex)); + connection.onClose(() => proc.kill()); + handleLogMessages(connection); + handleDiscoveryMessages(connection); + connection.listen(); + return connection; +} + +/** + * @param {import("vscode-jsonrpc").MessageConnection} connection + */ +function handleLogMessages(connection) { + connection.onNotification("log", (data) => { + switch (data.level) { + case "info": + // console.info('PET: ', data.message); + break; + case "warning": + console.warn("PET: ", data.message); + break; + case "error": + console.error("PET: ", data.message); + break; + case "debug": + // consol.debug('PET: ', data.message); + break; + default: + // console.trace('PET: ', data.message); + } + }); +} + +/** + * @param {import("vscode-jsonrpc").MessageConnection} connection + */ +function handleDiscoveryMessages(connection) { + connection.onNotification("manager", (mgr) => + console.log(`Discovered Manager (${mgr.tool}) ${mgr.executable}`) + ); + connection.onNotification("environment", (env) => { + environments.push(env); + }); +} + +/** + * Refresh the environment + * + * @param {import("vscode-jsonrpc").MessageConnection} connection + */ +async function refresh(connection) { + const configuration = { + // List of fully qualified paths to look for Environments, + // Generally this maps to workspace folders opened in the client, such as VS Code. + project_directories: [process.cwd()], + // List of fully qualified paths to look for virtual environments. + // Leave empty, if not applicable. + // In VS Code, users can configure custom locations where virtual environments are created. + environment_directories: [], + // Fully qualified path to the conda executable. + // Leave emtpy, if not applicable. + // In VS Code, users can provide the path to the executable as a hint to the location of where Conda is installed. + // Note: This should only be used if its known that PET is unable to find some Conda envs. + // However thats only a work around, ideally the issue should be reported to PET and fixed + conda_executable: undefined, + // Fully qualified path to the poetry executable. + // Leave emtpy, if not applicable. + // In VS Code, users can provide the path to the executable as a hint to the location of where Poetry is installed. + // Note: This should only be used if its known that PET is unable to find some Poetry envs. + // However thats only a work around, ideally the issue should be reported to PET and fixed + poetry_executable: undefined, + }; + + return connection + .sendRequest("refresh", configuration) + .then(({ duration }) => + console.log(`Found ${environments.length} environments in ${duration}ms.`) + ); +} + +/** + * Gets all possible information about the Python executable provided. + * This will spawn the Python executable (if not already done in the past). + * This must be used only if some of the information already avaialble is not sufficient. + * + * E.g. if a Python env was discovered and the version information is not know, + * but is requried, then call this method. + * If on the other hand, all of the information is already available, then there's no need to call this method. + * In fact it would be better to avoid calling this method, as it will spawn a new process & consume resouces. + * + * @param {String} executable Fully qualified path to the Python executable. + * @param {import("vscode-jsonrpc").MessageConnection} connection + */ +async function resolve(executable, connection) { + try { + const { environment, duration } = await connection.sendRequest( + "resolve", + executable + ); + console.log( + `Resolved (${environment.category}, ${environment.version}) ${environment.executable} in ${duration}ms` + ); + return environment; + } catch (ex) { + console.error(`Failed to resolve executable ${executable}`, ex.message); + } +} + +async function main() { + const connection = await start(); + await refresh(connection); + // Possible this env was discovered, and the version or prefix information is not known. + const env = await resolve("/usr/local/bin/python3", connection); + // Possible we have an enviornment that was never discovered and we need information about that. + const env2 = await resolve("/.venv/bin/python", connection); + + connection.end(); + process.exit(0); +} + +main(); From b80e04dbf1ff887c0542fc2b1b71d5f07dcd9c9d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 3 Jul 2024 13:16:58 +1000 Subject: [PATCH 2/8] Remove unwanted serialization layer --- crates/pet-reporter/src/environment.rs | 81 +------------------------- crates/pet-reporter/src/jsonrpc.rs | 1 - crates/pet-reporter/src/lib.rs | 1 - crates/pet-reporter/src/manager.rs | 33 ----------- crates/pet/src/jsonrpc.rs | 6 +- 5 files changed, 4 insertions(+), 118 deletions(-) delete mode 100644 crates/pet-reporter/src/manager.rs diff --git a/crates/pet-reporter/src/environment.rs b/crates/pet-reporter/src/environment.rs index 86a74f74..fd28c9dd 100644 --- a/crates/pet-reporter/src/environment.rs +++ b/crates/pet-reporter/src/environment.rs @@ -1,89 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -use crate::manager::Manager; use log::error; -use pet_core::{ - arch::Architecture, - python_environment::{PythonEnvironment, PythonEnvironmentCategory}, -}; -use serde::{Deserialize, Serialize}; +use pet_core::python_environment::{PythonEnvironment, PythonEnvironmentCategory}; use std::path::PathBuf; -// We want to maintain full control over serialization instead of relying on the enums or the like. -// Else its too easy to break the API by changing the enum variants. -fn python_category_to_string(category: &PythonEnvironmentCategory) -> &'static str { - match category { - PythonEnvironmentCategory::System => "system", - PythonEnvironmentCategory::MacCommandLineTools => "mac-command-line-tools", - PythonEnvironmentCategory::MacXCode => "mac-xcode", - PythonEnvironmentCategory::MacPythonOrg => "mac-python-org", - PythonEnvironmentCategory::GlobalPaths => "global-paths", - PythonEnvironmentCategory::Homebrew => "homebrew", - PythonEnvironmentCategory::Conda => "conda", - PythonEnvironmentCategory::LinuxGlobal => "linux-global", - PythonEnvironmentCategory::Pyenv => "pyenv", - PythonEnvironmentCategory::PyenvVirtualEnv => "pyenv-virtualenv", - PythonEnvironmentCategory::WindowsStore => "windows-store", - PythonEnvironmentCategory::WindowsRegistry => "windows-registry", - PythonEnvironmentCategory::Pipenv => "pipenv", - PythonEnvironmentCategory::VirtualEnvWrapper => "virtualenvwrapper", - PythonEnvironmentCategory::Venv => "venv", - PythonEnvironmentCategory::VirtualEnv => "virtualenv", - PythonEnvironmentCategory::Unknown => "unknown", - PythonEnvironmentCategory::Poetry => "poetry", - } -} - -// We want to maintain full control over serialization instead of relying on the enums or the like. -// Else its too easy to break the API by changing the enum variants. -fn architecture_to_string(arch: &Architecture) -> &'static str { - match arch { - Architecture::X64 => "x64", - Architecture::X86 => "x86", - } -} - -#[derive(Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -#[derive(Debug)] -pub struct Environment { - pub display_name: Option, - pub name: Option, - pub executable: Option, - pub category: String, - pub version: Option, - pub prefix: Option, - pub manager: Option, - pub project: Option, - pub arch: Option, - pub symlinks: Option>, -} - -impl Environment { - pub fn from(env: &PythonEnvironment) -> Environment { - Environment { - display_name: env.display_name.clone(), - name: env.name.clone(), - executable: env.executable.clone(), - category: python_category_to_string(&env.category).to_string(), - version: env.version.clone(), - prefix: env.prefix.clone(), - manager: match &env.manager { - Some(manager) => Manager::from(manager).into(), - None => None, - }, - project: env.project.clone(), - arch: env - .arch - .as_ref() - .map(architecture_to_string) - .map(|s| s.to_string()), - symlinks: env.symlinks.clone(), - } - } -} - pub fn get_environment_key(env: &PythonEnvironment) -> Option { if let Some(exe) = &env.executable { Some(exe.clone()) diff --git a/crates/pet-reporter/src/jsonrpc.rs b/crates/pet-reporter/src/jsonrpc.rs index d7d9964a..6a108acf 100644 --- a/crates/pet-reporter/src/jsonrpc.rs +++ b/crates/pet-reporter/src/jsonrpc.rs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::{environment::Environment, manager::Manager}; use env_logger::Builder; use log::{trace, LevelFilter}; use pet_core::{manager::EnvManager, python_environment::PythonEnvironment, reporter::Reporter}; diff --git a/crates/pet-reporter/src/lib.rs b/crates/pet-reporter/src/lib.rs index 0cc1f929..16c42605 100644 --- a/crates/pet-reporter/src/lib.rs +++ b/crates/pet-reporter/src/lib.rs @@ -4,6 +4,5 @@ pub mod cache; pub mod environment; pub mod jsonrpc; -pub mod manager; pub mod stdio; pub mod test; diff --git a/crates/pet-reporter/src/manager.rs b/crates/pet-reporter/src/manager.rs deleted file mode 100644 index 8760536d..00000000 --- a/crates/pet-reporter/src/manager.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use pet_core::manager::{EnvManager, EnvManagerType}; -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; - -fn tool_to_string(tool: &EnvManagerType) -> &'static str { - match tool { - EnvManagerType::Conda => "conda", - EnvManagerType::Pyenv => "pyenv", - EnvManagerType::Poetry => "poery", - } -} - -#[derive(Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -#[derive(Debug)] -pub struct Manager { - pub executable: PathBuf, - pub version: Option, - pub tool: String, -} - -impl Manager { - pub fn from(env: &EnvManager) -> Manager { - Manager { - executable: env.executable.clone(), - version: env.version.clone(), - tool: tool_to_string(&env.tool).to_string(), - } - } -} diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index 63a6bb20..bfe7f902 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -12,7 +12,7 @@ use pet_jsonrpc::{ send_error, send_reply, server::{start_server, HandlersKeyedByMethodName}, }; -use pet_reporter::{cache::CacheReporter, environment::Environment, jsonrpc}; +use pet_reporter::{cache::CacheReporter, jsonrpc}; use pet_telemetry::report_inaccuracies_identified_after_resolving; use serde::{Deserialize, Serialize}; use serde_json::{self, Value}; @@ -136,14 +136,14 @@ pub struct ResolveOptions { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ResolveResult { - environment: Environment, + environment: PythonEnvironment, duration: Option, } impl ResolveResult { fn new(env: &PythonEnvironment, duration: Result) -> ResolveResult { ResolveResult { - environment: Environment::from(env), + environment: env.clone(), duration: duration.ok().map(|d| d.as_millis()), } } From 1e8b828ccd3baf8a52905f30baaeadeff86e114e Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 15:19:31 +1000 Subject: [PATCH 3/8] Updates --- crates/pet-reporter/src/jsonrpc.rs | 4 ++-- crates/pet/src/find.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/pet-reporter/src/jsonrpc.rs b/crates/pet-reporter/src/jsonrpc.rs index 6a108acf..06c98442 100644 --- a/crates/pet-reporter/src/jsonrpc.rs +++ b/crates/pet-reporter/src/jsonrpc.rs @@ -12,12 +12,12 @@ pub struct JsonRpcReporter {} impl Reporter for JsonRpcReporter { fn report_manager(&self, manager: &EnvManager) { trace!("Reporting Manager {:?}", manager); - send_message("manager", Manager::from(manager).into()) + send_message("manager", manager.into()) } fn report_environment(&self, env: &PythonEnvironment) { trace!("Reporting Environment {:?}", env); - send_message("environment", Environment::from(env).into()) + send_message("environment", env.into()) } } diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index 8df7f72a..70516290 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use log::{info, trace, warn}; +use log::{trace, warn}; use pet_conda::CondaLocator; use pet_core::os_environment::{Environment, EnvironmentApi}; use pet_core::reporter::Reporter; @@ -45,7 +45,6 @@ pub fn find_and_report_envs( find_global_virtual_envs_time: Duration::from_secs(0), find_search_paths_time: Duration::from_secs(0), })); - info!("Started Refreshing Environments"); let start = std::time::Instant::now(); // From settings From 694663396b0e9c59b17d0bb1371ff8c3273a462a Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 15:58:45 +1000 Subject: [PATCH 4/8] udpates --- crates/pet/src/jsonrpc.rs | 42 +++++++++++---------------------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index bfe7f902..0293fc1d 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -4,10 +4,7 @@ use log::{error, info, trace}; use pet::resolve::resolve_environment; use pet_conda::Conda; -use pet_core::{ - os_environment::EnvironmentApi, python_environment::PythonEnvironment, reporter::Reporter, - Configuration, Locator, -}; +use pet_core::{os_environment::EnvironmentApi, reporter::Reporter, Configuration, Locator}; use pet_jsonrpc::{ send_error, send_reply, server::{start_server, HandlersKeyedByMethodName}, @@ -20,7 +17,7 @@ use std::{ path::PathBuf, sync::{Arc, RwLock}, thread, - time::{Duration, SystemTime, SystemTimeError}, + time::{Duration, SystemTime}, }; use crate::{find::find_and_report_envs, locators::create_locators}; @@ -134,21 +131,6 @@ pub struct ResolveOptions { pub executable: PathBuf, } -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ResolveResult { - environment: PythonEnvironment, - duration: Option, -} - -impl ResolveResult { - fn new(env: &PythonEnvironment, duration: Result) -> ResolveResult { - ResolveResult { - environment: env.clone(), - duration: duration.ok().map(|d| d.as_millis()), - } - } -} - pub fn handle_resolve(context: Arc, id: u32, params: Value) { match serde_json::from_value::(params.clone()) { Ok(request_options) => { @@ -175,20 +157,20 @@ pub fn handle_resolve(context: Arc, id: u32, params: Value) { &resolved, ); - trace!("Resolved env {:?} as {:?}", executable, resolved); - send_reply(id, Some(ResolveResult::new(&resolved, now.elapsed()))); + trace!( + "Resolved env ({:?}) {executable:?} as {resolved:?}", + now.elapsed() + ); + send_reply(id, resolved.into()); } else { error!( - "Failed to resolve env {:?}, returning discovered env {:?}", - executable, result.discovered - ); - send_reply( - id, - Some(ResolveResult::new(&result.discovered, now.elapsed())), + "Failed to resolve env {executable:?}, returning discovered env {:?}", + result.discovered ); + send_reply(id, result.discovered.into()); } } else { - error!("Failed to resolve env {:?}", executable); + error!("Failed to resolve env {executable:?}"); send_error( Some(id), -4, @@ -198,7 +180,7 @@ pub fn handle_resolve(context: Arc, id: u32, params: Value) { }); } Err(e) => { - error!("Failed to parse request {:?}: {}", params, e); + error!("Failed to parse request {params:?}: {e}"); send_error( Some(id), -4, From 3f6efc069b4155f8b4cab0ce8a38b00c68d6556b Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 16:20:53 +1000 Subject: [PATCH 5/8] Misc --- crates/pet-core/src/manager.rs | 4 +--- crates/pet-core/src/python_environment.rs | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/pet-core/src/manager.rs b/crates/pet-core/src/manager.rs index 6e4e3352..7acd294a 100644 --- a/crates/pet-core/src/manager.rs +++ b/crates/pet-core/src/manager.rs @@ -4,9 +4,7 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; -#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -#[derive(Debug, Hash)] +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq, Debug, Hash)] pub enum EnvManagerType { Conda, Poetry, diff --git a/crates/pet-core/src/python_environment.rs b/crates/pet-core/src/python_environment.rs index 88ef0059..f611e075 100644 --- a/crates/pet-core/src/python_environment.rs +++ b/crates/pet-core/src/python_environment.rs @@ -9,7 +9,6 @@ use std::path::PathBuf; use crate::{arch::Architecture, manager::EnvManager}; #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] #[derive(Debug, Hash)] pub enum PythonEnvironmentCategory { Conda, From e5b0c6522766d1f3078c76c60c51e49e73bfaf9c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 16:24:36 +1000 Subject: [PATCH 6/8] remove unwanted category --- crates/pet-core/src/python_environment.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/pet-core/src/python_environment.rs b/crates/pet-core/src/python_environment.rs index f611e075..3e54f411 100644 --- a/crates/pet-core/src/python_environment.rs +++ b/crates/pet-core/src/python_environment.rs @@ -8,8 +8,7 @@ use std::path::PathBuf; use crate::{arch::Architecture, manager::EnvManager}; -#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] -#[derive(Debug, Hash)] +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum PythonEnvironmentCategory { Conda, Homebrew, @@ -18,7 +17,6 @@ pub enum PythonEnvironmentCategory { PyenvVirtualEnv, // Pyenv virtualenvs. Pipenv, Poetry, - System, MacPythonOrg, MacCommandLineTools, LinuxGlobal, From d792428fa48839b291f74d7d798af5cbdaf41ae3 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 16:31:59 +1000 Subject: [PATCH 7/8] messages --- docs/JSONRPC.md | 242 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 docs/JSONRPC.md diff --git a/docs/JSONRPC.md b/docs/JSONRPC.md new file mode 100644 index 00000000..8dd8c36b --- /dev/null +++ b/docs/JSONRPC.md @@ -0,0 +1,242 @@ +# JSONPRC Messages + +The tool supports JSONRPC messages for communication. +The messages are sent over a stdio/stdout. The messages are in the form of a JSON object. + +The messages/notifications supported are listed below. + +This document assumes the reader is familiar with the JSONRPC 2.0 specification. +Hence there's no mention of the `jsonrpc` property in the messages. +For samples using JSONRPC, please have a look at the [sample.js](./sample.js) file. + +# Initialize/Configuration/Handshake Request + +At the moment there is no support for a configuration/handshake request. +The assumption is that all consumers of this tool require discovery of Python environments, hence the `refresh` method is currently treated as the initialization/handshake request. + +# Refresh Request + +This should always be the first request sent to the tool. +The request is expected to contain the configuraiton information for the tool to use. +All properties of the configuration are optional. + +_Request_: + +- method: `refresh` +- params: `RefreshParams` defined as below. + +_Response_: + +- result: `RefreshResult` defined as below. + +```typescript +interface RefreshParams { + /** + * This is a list of project directories. + * Useful for poetry, pipenv, virtualenvwrapper and the like to discover virtual environments that belong to specific project directories. + * E.g. `workspace folders` in vscode. + * + * If not provided, then environments such as poetry, pipenv, and the like will not be reported. + * This is because poetry, pipenv, and the like are project specific enviornents. + */ + project_directories:? string[]; + /** + * This is a list of directories where we should look for virtual environments. + * This is useful when the virtual environments are stored in some custom locations. + * + * Useful for VS Code so users can configure where they store virtual environments. + */ + environment_directories: getCustomVirtualEnvDirs(), + /** + * This is the path to the conda executable. + * If conda is installed in the usual location, there's no need to update this value. + * + * Useful for VS Code so users can configure where they have installed Conda. + */ + conda_executable: getPythonSettingAndUntildify(CONDAPATH_SETTING_KEY), + /** + * This is the path to the conda executable. + * If Poetry is installed in the usual location, there's no need to update this value. + * + * Useful for VS Code so users can configure where they have installed Poetry. + */ + poetry_executable: getPythonSettingAndUntildify('poetryPath'), +} + +interface RefreshResult { + /** + * Total time taken to refresh the list of Python environments. + * Duration is in milliseconds. + */ + duration: number; +} +``` + +# Resolve Request + +Use this request to resolve a Python environment from a given Python path. +Note: This request will generally end up spawning the Python process to get the environment information. +Hence it is advisable to use this request sparingly and rely on Python environments being discovered or relying on the information returned by the `refresh` request. + +_Request_: + +- method: `resolve` +- params: `ResolveParams` defined as below. + +_Response_: + +- result: `Environment` defined as below. + +```typescript +interface ResolveParams { + /** + * The fully qualified path to the Pyton executable. + */ + executable: string; +} + +interface Environment { + /** + * The display name of the enviornment. + * Generally empty, however some tools such as Windows Registry may provide a display name. + */ + disdplay_name?: string; + /** + * The name of the envirionment. + * Generally empty, however some tools such as Conda may provide a display name. + * In the case of conda, this is the name of the conda environment and is used in activation of the conda environment. + */ + name?: string; + /** + * The fully qualified path to the executable of the envirionment. + * Generally non-empty, however in the case of conda environmentat that do not have Python installed in them, this may be empty. + * + * Some times this may not be the same as the `sys.executable` retured by the Python runtime. + * This is because this path is the shortest and/or most user friendly path to the Python executable. + * For instance its simpler for users to remember and use /usr/local/bin/python3 as opposed to /Library/Frameworks/Python.framework/Versions/Current/bin/python3 + * + * All known symlinks to the executable are returned in the `symlinks` property. + */ + executable?: string; + /** + * The type/category of the environment. + * + * If an environment is discovered and the kind is not know, then `Unknown` is used. + * I.e this is never Optional. + */ + category: + | "Conda" // Conda environment + | "Homebrew" // Homebrew installed Python + | "Pyenv" // Pyenv installed Python + | "GlobalPaths" // Unknown Pyton environment, found in the PATH environment variable + | "PyenvVirtualEnv" // pyenv-virtualenv environment + | "Pipenv" // Pipenv environment + | "Poetry" // Poetry environment + | "MacPythonOrg" // Python installed from python.org on Mac + | "MacCommandLineTools" // Python installed from the Mac command line tools + | "LinuxGlobal" // Python installed from the system package manager on Linux + | "MacXCode" // Python installed from XCode on Mac + | "Unknown" // Unknown Python environment + | "Venv" // Python venv environment (generally created using the `venv` module) + | "VirtualEnv" // Python virtual environment + | "VirtualEnvWrapper" // Virtualenvwrapper Environment + | "WindowsStore" // Python installed from the Windows Store + | "WindowsRegistry"; // Python installed & found in Windows Registry + /** + * The version of the python executable. + * This will at a minimum contain the 3 parts of the version such as `3.8.1`. + * Somtime it might also contain other parts of the version such as `3.8.1+` or `3.8.1.final.0` + */ + version?: string; + /** + * The prefix of the Python environment as returned by `sys.prefix` in the Python runtime. + */ + prefix?: string; + /** + * The bitness of the Python environment. + */ + arch?: "x64" | "x86"; + /** + * The list of known symlinks to the Python executable. + * Note: These are not all the symlinks, but only the known ones. + * & they might not necessarily be symlinks as known in the strict sense, however they are all the known executables that point to the same Python Environment. + * + * E.g. the exes /bin/python and /bin/python3 are symlinks to the same Python environment. + */ + symlinks?: string; + /** + * The project folder this Python environment belongs to. + * Poetry, Pipenv, Virtualenvwrapper and the like are project specific environments. + * This is the folder where the project is located. + */ + project?: string; + /** + * The associated manager. + * E.g. `poetry`, `conda`, `pyenv` and the like. + * + * Even if a conda environment is discovered, the manager can still be empty. + * This happens when we're unable to determine the manager associated with the environment. + * + * Note, just because this tool discoveres other conda environments and they all have managers associated with them, it does not mean that we can use the same manager for this environment when not know. + * Thats because there could be multiple conda installations on the system, hence we try not to make any assumptions. + */ + manager?: Manager; +} + +interface Manager { + /** + * The fully qualified path to the executable of the manager. + * E.g. fully qualified path to the conda exe. + */ + executable: string; + /** + * The type of the Manager. + */ + tool?: "conda" | "poetry" | "pyenv"; + /** + * The version of the manager/tool. + * In the case of conda, this is the version of conda. + */ + version?: string; +} +``` + +# Log Notification + +Sent by the server to log messages + +_Notification_: + +- method: `resolve` +- params: `LogParams` defined as below. + +```typescript +interface LogParams { + /** + * The level of the log message. + */ + level: "info" | "warning" | "error" | "debug" | "trace"; + /** + * Message to log. + */ + message: string; +} +``` + +# Manager Notification + +Sent by the server whenever an Environment Manager is discovered. + +_Notification_: + +- method: `manager` +- params: `Manager` defined earlier. + +# Environment Notification + +Sent by the server whenever an Environment is discovered. + +_Notification_: + +- method: `environment` +- params: `Environment` defined earlier. From 116689502d47fa3ba7a0407678b77746fba74a3d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 4 Jul 2024 16:41:18 +1000 Subject: [PATCH 8/8] oops --- docs/JSONRPC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/JSONRPC.md b/docs/JSONRPC.md index 8dd8c36b..c6ec16fa 100644 --- a/docs/JSONRPC.md +++ b/docs/JSONRPC.md @@ -192,7 +192,7 @@ interface Manager { /** * The type of the Manager. */ - tool?: "conda" | "poetry" | "pyenv"; + tool: "conda" | "poetry" | "pyenv"; /** * The version of the manager/tool. * In the case of conda, this is the version of conda.