From 1ee81f93841cb1e4ce2f988f599fff7b3876fd5a Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Fri, 11 Jul 2025 11:24:55 -0400 Subject: [PATCH 01/40] Fix one-off error in combine_selectors This caused problems for single-node trees --- src/coniferest/evaluator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index d45f72b..e46fbf0 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -108,10 +108,11 @@ def combine_selectors(cls, selectors_list): # Assign a unique sequential index to every leaf # The index is used for weighted scores leaf_mask = selectors["feature"] < 0 - leaf_count = np.count_nonzero(leaf_mask) - leaf_offsets = np.full_like(node_offsets, leaf_count) - leaf_offsets[:-1] = np.cumsum(leaf_mask)[node_offsets[:-1]] + # Each offset tells how many leafs are in all previous trees + leaf_offsets = np.zeros_like(node_offsets) + leaf_offsets[1:] = np.cumsum(leaf_mask)[node_offsets[1:] - 1] + leaf_count = leaf_offsets[-1] selectors["left"][leaf_mask] = np.arange(0, leaf_count) From 2c0dad4c4b8d06dd277b83ed740d854cda03167c Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 15 May 2024 14:47:16 -0300 Subject: [PATCH 02/40] Initial re-impl of Cython code in Rust --- pyproject.toml | 4 +- rust/Cargo.lock | 250 ++++++++++++++++------- rust/Cargo.toml | 22 +-- rust/src/lib.rs | 385 ++++++++++++++++++++++++++++++++++-- src/coniferest/evaluator.py | 22 +-- 5 files changed, 571 insertions(+), 112 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 12223c0..13a91ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ build-backend = "maturin" [project] name = "coniferest" +version = "0.0.14" description = "Coniferous forests for better machine learning" readme = "README.md" requires-python = ">=3.10" @@ -28,7 +29,6 @@ dependencies = [ "scikit-learn>=1.4,<2", "onnxconverter-common", ] -dynamic = ["version"] [project.optional-dependencies] datasets = [ @@ -47,7 +47,7 @@ dev = [ "Source Code" = "https://github.com/snad-space/coniferest" [tool.maturin] -module-name = "coniferest.calc_trees" +module-name = "coniferest.calc_paths_sum" # It asks to use Cargo.lock to make the build reproducible locked = true diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4e6641c..4e93336 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,16 +1,28 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 4 +version = 3 [[package]] name = "autocfg" -version = "1.5.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "coniferest" -version = "0.1.0" +version = "0.0.14" dependencies = [ "enum_dispatch", "itertools", @@ -23,9 +35,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -42,15 +54,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "either" -version = "1.15.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "enum_dispatch" @@ -66,36 +78,46 @@ dependencies = [ [[package]] name = "heck" -version = "0.5.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "indoc" -version = "2.0.6" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "itertools" -version = "0.14.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "libc" -version = "0.2.174" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "lock_api" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] [[package]] name = "matrixmultiply" -version = "0.3.10" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" dependencies = [ "autocfg", "rawpointer", @@ -103,34 +125,32 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "ndarray" -version = "0.16.1" +version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" dependencies = [ "matrixmultiply", "num-complex", "num-integer", "num-traits", - "portable-atomic", - "portable-atomic-util", "rawpointer", "rayon", ] [[package]] name = "num-complex" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] @@ -146,18 +166,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.19" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] [[package]] name = "numpy" -version = "0.25.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f1dee9aa8d3f6f8e8b9af3803006101bb3653866ef056d530d53ae68587191" +checksum = "ec170733ca37175f5d75a5bea5911d6ff45d2cd52849ce98b685394e4f2f37f4" dependencies = [ "libc", "ndarray", @@ -165,50 +185,64 @@ dependencies = [ "num-integer", "num-traits", "pyo3", - "pyo3-build-config", "rustc-hash", ] [[package]] name = "once_cell" -version = "1.21.3" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] -name = "portable-atomic" -version = "1.11.1" +name = "parking_lot" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] [[package]] -name = "portable-atomic-util" -version = "0.2.4" +name = "parking_lot_core" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ - "portable-atomic", + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.25.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ + "cfg-if", "indoc", "libc", "memoffset", - "once_cell", + "parking_lot", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -218,9 +252,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.25.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -228,9 +262,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.25.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -238,9 +272,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.25.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -250,9 +284,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.25.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck", "proc-macro2", @@ -263,9 +297,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.40" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -278,9 +312,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" dependencies = [ "either", "rayon-core", @@ -296,17 +330,38 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags", +] + [[package]] name = "rustc-hash" -version = "2.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "syn" -version = "2.0.104" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", "quote", @@ -315,18 +370,75 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unindent" -version = "0.2.4" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 283e703..78824da 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,31 +1,23 @@ [package] name = "coniferest" -version = "0.1.0" +version = "0.0.16" edition = "2021" [lib] -name = "calc_trees" +name = "coniferest" crate-type = ["cdylib"] # We'd like to build fast code with `pip install -e '.[dev]'` [profile.dev] opt-level = 3 -# Makes linking slower, but the resulting extension module is faster -[profile.release] -lto = true -codegen-units = 1 - -[features] -default = ["pyo3/abi3-py310"] - [dependencies] enum_dispatch = "0.3" -itertools = "0.14" -pyo3 = { version = "0.25", features = ["extension-module"] } +itertools = "0.12" +pyo3 = { version = "0.21", features = ["abi3-py39", "extension-module"] } # Needs to be consistent with ndarray dependecy in numpy -ndarray = { version = "0.16", features = ["rayon"] } +ndarray = { version = "0.15", features = ["rayon"] } num-traits = "0.2" -numpy = "0.25" +numpy = "0.21" # Needs to be consistent with rayon dependecy in ndarray -rayon = "1.10" +rayon = "1.9" diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 23df59e..9ca6dcb 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,19 +1,378 @@ -mod mut_slices; -mod selector; -mod tree_traversal; - -use crate::selector::Selector; -use crate::tree_traversal::{ - calc_apply, calc_feature_delta_sum, calc_paths_sum, calc_paths_sum_transpose, -}; +use enum_dispatch::enum_dispatch; +use itertools::Itertools; +use ndarray::{Array1, ArrayView1, ArrayView2, Axis, Zip}; +use num_traits::AsPrimitive; +use numpy::PyArrayMethods; +use numpy::{Element, PyArray, PyArrayDescr}; +use numpy::{PyArray1, PyArray2}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use pyo3::py_run; +use pyo3::types::PyDict; +use rayon::prelude::*; +use std::iter; +use std::sync::{Arc, Mutex}; -#[pymodule(gil_used = false)] -fn calc_trees(py: Python, m: &Bound) -> PyResult<()> { - m.add("selector_dtype", Selector::dtype(py)?)?; +/// Selector is the representation of decision tree nodes: either branches or leafs. +/// +/// We use "C"-representation with standard alignment (np.dtype(align=True)), but "packed" +/// (dtype(aligh=False)) would work as well. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub(crate) struct Selector { + /// Feature index to branch on, -1.0 if leaf + feature: i32, + /// Index of left subtree, leaf_id if leaf + left: i32, + /// Feature value to branch on, resulting decision score if leaf + value: f64, + /// Index of right subtree, -1 if leaf + right: i32, + /// Natural logarithm of the number of samples in the node + log_n_node_samples: f32, +} + +impl Selector { + pub(crate) fn dtype(py: Python) -> PyResult> { + let locals = PyDict::new_bound(py); + py_run!( + py, + *locals, + r#" + dtype = __import__('numpy').dtype( + [('feature', 'i4'), ('left', 'i4'), ('value', 'f8'), ('right', 'i4')], + align=True, + ) + "# + ); + Ok(locals + .get_item("dtype") + .expect("Error in built-in Python code for dtype initialization") + .expect("Error in built-in Python code for dtype initialization: dtype cannot be None") + .downcast::()? + .clone()) + } + + #[inline(always)] + pub(crate) fn is_leaf(&self) -> bool { + self.feature == -1 + } +} + +/// Implementation of [numpy::Element] for [Selector] +/// +/// Safety: we guarantee that [Selector] has the same layout as it would have in numpy with +/// [Selector::dtype] +unsafe impl Element for Selector { + const IS_COPY: bool = true; + + fn get_dtype_bound(py: Python) -> Bound { + Self::dtype(py).unwrap() + } +} + +#[enum_dispatch] +trait DataTrait<'py> { + fn calc_paths_sum( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + weights: Option>>, + num_threads: usize, + ) -> PyResult>>; + + fn calc_paths_sum_transpose( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + leaf_count: usize, + weights: Option>>, + num_threads: usize, + ) -> PyResult>>; +} + +impl<'py, T> DataTrait<'py> for Bound<'py, PyArray2> +where + T: Element + Copy + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + fn calc_paths_sum( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + weights: Option>>, + num_threads: usize, + ) -> PyResult>> { + let selectors = selectors.readonly(); + let selectors_view = selectors.as_array(); + check_selectors(selectors_view)?; + + let indices = indices.readonly(); + let indices_view = indices.as_array(); + check_indices(indices_view, selectors.len()?)?; + + let data = self.readonly(); + let data_view = data.as_array(); + check_data(data_view)?; + + let weights = weights.map(|weights| weights.readonly()); + let weights_view = weights.as_ref().map(|weights| weights.as_array()); + + // Here we need to dispatch `data` and run the template function + let values = calc_paths_sum_impl( + selectors_view, + indices_view, + data_view, + weights_view, + num_threads, + ); + Ok(PyArray::from_owned_array_bound(py, values)) + } + + fn calc_paths_sum_transpose( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + leaf_count: usize, + weights: Option>>, + num_threads: usize, + ) -> PyResult>> { + let selectors = selectors.readonly(); + let selectors_view = selectors.as_array(); + crate::check_selectors(selectors_view)?; + + let indices = indices.readonly(); + let indices_view = indices.as_array(); + crate::check_indices(indices_view, selectors.len()?)?; + + let data = self.readonly(); + let data_view = data.as_array(); + crate::check_data(data_view)?; + + let weights = weights.map(|weights| weights.readonly()); + let weights_view = weights.as_ref().map(|weights| weights.as_array()); + + // Here we need to dispatch `data` and run the template function + let values = crate::calc_paths_sum_transpose_impl( + selectors_view, + indices_view, + leaf_count, + data_view, + weights_view, + num_threads, + ); + Ok(PyArray::from_owned_array_bound(py, values)) + } +} + +#[enum_dispatch(DataTrait)] +#[derive(FromPyObject)] +enum Data<'py> { + F64(Bound<'py, PyArray2>), + F32(Bound<'py, PyArray2>), +} + +// It looks like the performance is not affected by returning a copy of Selector, not reference. +#[inline] +fn find_leaf(tree: &[Selector], sample: &[T]) -> Selector +where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let mut i = 0; + loop { + let selector = *unsafe { tree.get_unchecked(i) }; + if selector.is_leaf() { + break selector; + } + + // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? + let threshold: T = selector.value.as_(); + i = if *unsafe { sample.get_unchecked(selector.feature as usize) } <= threshold { + selector.left as usize + } else { + selector.right as usize + }; + } +} + +#[inline] +fn check_selectors(selectors: ArrayView1) -> PyResult<()> { + if !selectors.is_standard_layout() { + return Err(PyValueError::new_err( + "selectors must be contiguous and in memory order", + )); + } + Ok(()) +} + +#[inline] +fn check_indices(indices: ArrayView1, selectors_length: usize) -> PyResult<()> { + if let Some(indices) = indices.as_slice() { + for (x, y) in indices.iter().copied().tuple_windows() { + if x > y { + return Err(PyValueError::new_err( + "indices must be sorted in ascending order", + )); + } + } + if indices[indices.len() - 1] as usize > selectors_length { + return Err(PyValueError::new_err( + "indices are out of range of the selectors", + )); + } + Ok(()) + } else { + Err(PyValueError::new_err( + "indices must be contiguous and in memory order", + )) + } +} + +#[inline] +fn check_data(data: ArrayView2) -> PyResult<()> { + if !data.is_standard_layout() { + return Err(PyValueError::new_err( + "data must be contiguous and in memory order", + )); + } + Ok(()) +} + +#[pyfunction] +#[pyo3(signature = (selectors, indices, data, weights = None, num_threads = 0))] +pub(crate) fn calc_paths_sum<'py>( + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + // TODO: support f32 data + data: Data<'py>, + weights: Option>>, + num_threads: usize, +) -> PyResult>> { + data.calc_paths_sum(py, selectors, indices, weights, num_threads) +} + +fn calc_paths_sum_impl( + selectors: ArrayView1, + indices: ArrayView1, + data: ArrayView2, + weights: Option>, + num_threads: usize, +) -> Array1 +where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let mut paths = Array1::zeros(data.nrows()); + + let indices = indices.as_slice().unwrap(); + let selectors = selectors.as_slice().unwrap(); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + Zip::from(paths.view_mut()) + .and(data.rows()) + .par_for_each(|path, sample| { + for (tree_start, tree_end) in + indices.iter().map(|i| *i as usize).tuple_windows() + { + let tree_selectors = + unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + if let Some(weights) = weights { + *path += *unsafe { weights.uget(leaf.left as usize) } * leaf.value; + } else { + *path += leaf.value; + } + } + }) + }); + + paths +} + +#[pyfunction] +#[pyo3(signature = (selectors, indices, data, leaf_count, weights = None, num_threads = 0))] +pub(crate) fn calc_paths_sum_transpose<'py>( + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + data: Data<'py>, + leaf_count: usize, + weights: Option>>, + num_threads: usize, +) -> PyResult>> { + data.calc_paths_sum_transpose(py, selectors, indices, leaf_count, weights, num_threads) +} + +fn calc_paths_sum_transpose_impl( + selectors: ArrayView1, + indices: ArrayView1, + leaf_count: usize, + data: ArrayView2, + weights: Option>, + num_threads: usize, +) -> Array1 +where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + // We need leaf_offsets instead of leaf_counts here. + // It would allow to split the array and write safely from multiple threads. + let values = Arc::new((0..leaf_count).map(|_| Mutex::new(0.0)).collect::>()); + + let selectors = selectors.as_slice().unwrap(); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + indices + .iter() + .map(|i| *i as usize) + .tuple_windows() + .zip(iter::repeat_with(|| values.clone())) + .par_bridge() + .for_each(|((tree_start, tree_end), values)| { + for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { + let tree_selectors = + unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + let mut value = values[leaf.left as usize].lock().unwrap(); + if let Some(weights) = weights { + *value += weights[x_index] * leaf.value; + } else { + *value += leaf.value; + } + } + }) + }); + + Arc::try_unwrap(values) + .unwrap() + .into_iter() + .map(|mutex| mutex.into_inner().unwrap()) + .collect() +} + +#[pymodule] +#[pyo3(name = "calc_paths_sum")] +fn rust_module(_py: Python, m: &Bound) -> PyResult<()> { + m.add("selector_dtype", Selector::dtype(_py)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum, m)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum_transpose, m)?)?; - m.add_function(wrap_pyfunction!(calc_feature_delta_sum, m)?)?; - m.add_function(wrap_pyfunction!(calc_apply, m)?)?; Ok(()) } diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index e46fbf0..84878d4 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -2,7 +2,7 @@ import numpy as np -from .calc_trees import calc_apply, calc_feature_delta_sum, calc_paths_sum, selector_dtype # noqa +from .calc_paths_sum import calc_paths_sum, selector_dtype # noqa from .utils import average_path_length __all__ = ["ForestEvaluator"] @@ -11,7 +11,7 @@ class ForestEvaluator: selector_dtype = selector_dtype - def __init__(self, samples, selectors, node_offsets, leaf_offsets, *, num_threads, sampletrees_per_batch): + def __init__(self, samples, selectors, indices, leaf_count, *, num_threads): """ Base class for the forest evaluators. Does the trivial job: * runs calc_paths_sum written in Rust, @@ -103,7 +103,7 @@ def combine_selectors(cls, selectors_list): node_offsets[1:] = np.add.accumulate(lens) for i in range(len(selectors_list)): - selectors[node_offsets[i] : node_offsets[i + 1]] = selectors_list[i] + selectors[indices[i]: indices[i + 1]] = selectors_list[i] # Assign a unique sequential index to every leaf # The index is used for weighted scores @@ -143,17 +143,11 @@ def score_samples(self, x): x = np.ascontiguousarray(x) return -( - 2 - ** ( - -calc_paths_sum( - self.selectors, - self.node_offsets, - x, - num_threads=self.num_threads, - batch_size=self.get_batch_size(self.n_trees), + 2 + ** ( + -calc_paths_sum(self.selectors, self.indices, x, num_threads=self.num_threads) + / (self.average_path_length(self.samples) * trees) ) - / (self.average_path_length(self.samples) * self.n_trees) - ) ) def _feature_delta_sum(self, x): @@ -179,6 +173,8 @@ def feature_importance(self, x): return np.sum(delta_sum, axis=0) / np.sum(hit_count, axis=0) / self.average_path_length(self.samples) def apply(self, x): + raise NotImplemented("Not implemented in Rust yet") + if not x.flags["C_CONTIGUOUS"]: x = np.ascontiguousarray(x) From 6deadb674b2d8a69c81582c7ce66a86c5638382e Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 15 May 2024 15:01:26 -0300 Subject: [PATCH 03/40] Fix selector's dtype --- rust/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 9ca6dcb..9e5c40c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -40,7 +40,13 @@ impl Selector { *locals, r#" dtype = __import__('numpy').dtype( - [('feature', 'i4'), ('left', 'i4'), ('value', 'f8'), ('right', 'i4')], + [ + ('feature', 'i4'), + ('left', 'i4'), + ('value', 'f8'), + ('right', 'i4'), + ('log_n_node_samples', 'f4') + ], align=True, ) "# From 72f328b65b0b1dabcabf92dbba68b4640c74e911 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 15 May 2024 16:23:52 -0300 Subject: [PATCH 04/40] Rust re-impl of feature_delta_sum --- rust/src/lib.rs | 121 ++++++++++++++++++++++++++++++++++-- src/coniferest/evaluator.py | 2 +- 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 9e5c40c..c43cf23 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,6 +1,6 @@ use enum_dispatch::enum_dispatch; use itertools::Itertools; -use ndarray::{Array1, ArrayView1, ArrayView2, Axis, Zip}; +use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis, Zip}; use num_traits::AsPrimitive; use numpy::PyArrayMethods; use numpy::{Element, PyArray, PyArrayDescr}; @@ -97,6 +97,14 @@ trait DataTrait<'py> { weights: Option>>, num_threads: usize, ) -> PyResult>>; + + fn calc_feature_delta_sum( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + num_threads: usize, + ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)>; } impl<'py, T> DataTrait<'py> for Bound<'py, PyArray2> @@ -149,15 +157,15 @@ where ) -> PyResult>> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); - crate::check_selectors(selectors_view)?; + check_selectors(selectors_view)?; let indices = indices.readonly(); let indices_view = indices.as_array(); - crate::check_indices(indices_view, selectors.len()?)?; + check_indices(indices_view, selectors.len()?)?; let data = self.readonly(); let data_view = data.as_array(); - crate::check_data(data_view)?; + check_data(data_view)?; let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); @@ -173,6 +181,34 @@ where ); Ok(PyArray::from_owned_array_bound(py, values)) } + + fn calc_feature_delta_sum( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + num_threads: usize, + ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { + let selectors = selectors.readonly(); + let selectors_view = selectors.as_array(); + check_selectors(selectors_view)?; + + let indices = indices.readonly(); + let indices_view = indices.as_array(); + check_indices(indices_view, selectors.len()?)?; + + let data = self.readonly(); + let data_view = data.as_array(); + check_data(data_view)?; + + let (delta_sum, hit_count) = + calc_feature_delta_sum_impl(selectors_view, indices_view, data_view, num_threads); + + let delta_sum = PyArray::from_owned_array_bound(py, delta_sum); + let hit_count = PyArray::from_owned_array_bound(py, hit_count); + + Ok((delta_sum, hit_count)) + } } #[enum_dispatch(DataTrait)] @@ -374,11 +410,88 @@ where .collect() } +#[pyfunction] +#[pyo3(signature = (selectors, indices, data, num_threads = 0))] +pub(crate) fn calc_feature_delta_sum<'py>( + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + indices: Bound<'py, PyArray1>, + data: Data<'py>, + num_threads: usize, +) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { + data.calc_feature_delta_sum(py, selectors, indices, num_threads) +} + +fn calc_feature_delta_sum_impl( + selectors: ArrayView1, + indices: ArrayView1, + data: ArrayView2, + num_threads: usize, +) -> (Array2, Array2) +where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let indices = indices.as_slice().unwrap(); + let selectors = selectors.as_slice().unwrap(); + + let mut delta_sum = Array2::zeros((data.nrows(), data.ncols())); + let mut hit_count = Array2::zeros((data.nrows(), data.ncols())); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + Zip::from(data.rows()) + .and(delta_sum.rows_mut()) + .and(hit_count.rows_mut()) + .par_for_each(|sample, mut delta_sum_row, mut hit_count_row| { + for (tree_start, tree_end) in + indices.iter().map(|i| *i as usize).tuple_windows() + { + let tree_selectors = + unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let mut i = 0; + let mut parent_selector: &Selector; + loop { + parent_selector = unsafe { tree_selectors.get_unchecked(i) }; + if parent_selector.is_leaf() { + break; + } + + // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? + let threshold: T = parent_selector.value.as_(); + i = if *unsafe { sample.uget(parent_selector.feature as usize) } + <= threshold + { + parent_selector.left as usize + } else { + parent_selector.right as usize + }; + + let child_selector = unsafe { tree_selectors.get_unchecked(i) }; + *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += + 1.0 + 2.0 + * (child_selector.log_n_node_samples as f64 + - parent_selector.log_n_node_samples as f64); + *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += + 1; + } + } + }); + }); + + (delta_sum, hit_count) +} + #[pymodule] #[pyo3(name = "calc_paths_sum")] fn rust_module(_py: Python, m: &Bound) -> PyResult<()> { m.add("selector_dtype", Selector::dtype(_py)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum, m)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum_transpose, m)?)?; + m.add_function(wrap_pyfunction!(calc_feature_delta_sum, m)?)?; Ok(()) } diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index 84878d4..5bc2444 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -2,7 +2,7 @@ import numpy as np -from .calc_paths_sum import calc_paths_sum, selector_dtype # noqa +from .calc_paths_sum import calc_feature_delta_sum, calc_paths_sum, selector_dtype # noqa from .utils import average_path_length __all__ = ["ForestEvaluator"] From d18e3aa4e1c788385be3172da73777acf57ef67d Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 15 May 2024 19:34:52 -0300 Subject: [PATCH 05/40] leaf_offsets - leaf_offsets instead of leaf_count - indices is renamed to node_offsets # Conflicts: # src/coniferest/aadforest.py --- rust/src/lib.rs | 165 ++++++++++++++++----------- rust/src/mut_slices.rs | 220 +++--------------------------------- src/coniferest/aadforest.py | 16 +-- src/coniferest/evaluator.py | 23 ++-- 4 files changed, 127 insertions(+), 297 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index c43cf23..b5095a9 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,3 +1,5 @@ +mod mut_slices; + use enum_dispatch::enum_dispatch; use itertools::Itertools; use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis, Zip}; @@ -10,8 +12,6 @@ use pyo3::prelude::*; use pyo3::py_run; use pyo3::types::PyDict; use rayon::prelude::*; -use std::iter; -use std::sync::{Arc, Mutex}; /// Selector is the representation of decision tree nodes: either branches or leafs. /// @@ -83,7 +83,7 @@ trait DataTrait<'py> { &self, py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, ) -> PyResult>>; @@ -92,8 +92,8 @@ trait DataTrait<'py> { &self, py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, - leaf_count: usize, + node_offsets: Bound<'py, PyArray1>, + leaf_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, ) -> PyResult>>; @@ -102,7 +102,7 @@ trait DataTrait<'py> { &self, py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, num_threads: usize, ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)>; } @@ -116,7 +116,7 @@ where &self, py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, ) -> PyResult>> { @@ -124,9 +124,9 @@ where let selectors_view = selectors.as_array(); check_selectors(selectors_view)?; - let indices = indices.readonly(); - let indices_view = indices.as_array(); - check_indices(indices_view, selectors.len()?)?; + let node_offsets = node_offsets.readonly(); + let node_offsets_view = node_offsets.as_array(); + check_node_offsets(node_offsets_view, selectors.len()?)?; let data = self.readonly(); let data_view = data.as_array(); @@ -138,7 +138,7 @@ where // Here we need to dispatch `data` and run the template function let values = calc_paths_sum_impl( selectors_view, - indices_view, + node_offsets_view, data_view, weights_view, num_threads, @@ -150,8 +150,8 @@ where &self, py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, - leaf_count: usize, + node_offsets: Bound<'py, PyArray1>, + leaf_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, ) -> PyResult>> { @@ -159,9 +159,13 @@ where let selectors_view = selectors.as_array(); check_selectors(selectors_view)?; - let indices = indices.readonly(); - let indices_view = indices.as_array(); - check_indices(indices_view, selectors.len()?)?; + let node_offsets = node_offsets.readonly(); + let node_offsets_view = node_offsets.as_array(); + check_node_offsets(node_offsets_view, selectors_view.len())?; + + let leaf_offsets = leaf_offsets.readonly(); + let leaf_offsets_view = leaf_offsets.as_array(); + check_leaf_offsets(leaf_offsets_view, node_offsets_view.len())?; let data = self.readonly(); let data_view = data.as_array(); @@ -171,10 +175,10 @@ where let weights_view = weights.as_ref().map(|weights| weights.as_array()); // Here we need to dispatch `data` and run the template function - let values = crate::calc_paths_sum_transpose_impl( + let values = calc_paths_sum_transpose_impl( selectors_view, - indices_view, - leaf_count, + node_offsets_view, + leaf_offsets_view, data_view, weights_view, num_threads, @@ -186,23 +190,23 @@ where &self, py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, num_threads: usize, ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); check_selectors(selectors_view)?; - let indices = indices.readonly(); - let indices_view = indices.as_array(); - check_indices(indices_view, selectors.len()?)?; + let node_offsets = node_offsets.readonly(); + let node_offsets_view = node_offsets.as_array(); + check_node_offsets(node_offsets_view, selectors.len()?)?; let data = self.readonly(); let data_view = data.as_array(); check_data(data_view)?; let (delta_sum, hit_count) = - calc_feature_delta_sum_impl(selectors_view, indices_view, data_view, num_threads); + calc_feature_delta_sum_impl(selectors_view, node_offsets_view, data_view, num_threads); let delta_sum = PyArray::from_owned_array_bound(py, delta_sum); let hit_count = PyArray::from_owned_array_bound(py, hit_count); @@ -253,24 +257,47 @@ fn check_selectors(selectors: ArrayView1) -> PyResult<()> { } #[inline] -fn check_indices(indices: ArrayView1, selectors_length: usize) -> PyResult<()> { - if let Some(indices) = indices.as_slice() { - for (x, y) in indices.iter().copied().tuple_windows() { +fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) -> PyResult<()> { + if let Some(node_offsets) = node_offsets.as_slice() { + for (x, y) in node_offsets.iter().copied().tuple_windows() { if x > y { return Err(PyValueError::new_err( - "indices must be sorted in ascending order", + "node_offsets must be sorted in ascending order", )); } } - if indices[indices.len() - 1] as usize > selectors_length { + if node_offsets[node_offsets.len() - 1] as usize > selectors_length { return Err(PyValueError::new_err( - "indices are out of range of the selectors", + "node_offsets are out of range of the selectors", )); } Ok(()) } else { Err(PyValueError::new_err( - "indices must be contiguous and in memory order", + "node_offsets must be contiguous and in memory order", + )) + } +} + +#[inline] +fn check_leaf_offsets(leaf_offsets: ArrayView1, node_offset_len: usize) -> PyResult<()> { + if leaf_offsets.len() != node_offset_len { + return Err(PyValueError::new_err( + "leaf_offsets must have the same length as node_offsets", + )); + } + if let Some(leaf_offsets) = leaf_offsets.as_slice() { + for (x, y) in leaf_offsets.iter().copied().tuple_windows() { + if x > y { + return Err(PyValueError::new_err( + "leaf_offsets must be sorted in ascending order", + )); + } + } + Ok(()) + } else { + Err(PyValueError::new_err( + "leaf_offsets must be contiguous and in memory order", )) } } @@ -286,22 +313,22 @@ fn check_data(data: ArrayView2) -> PyResult<()> { } #[pyfunction] -#[pyo3(signature = (selectors, indices, data, weights = None, num_threads = 0))] +#[pyo3(signature = (selectors, node_offsets, data, weights = None, num_threads = 0))] pub(crate) fn calc_paths_sum<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, // TODO: support f32 data data: Data<'py>, weights: Option>>, num_threads: usize, ) -> PyResult>> { - data.calc_paths_sum(py, selectors, indices, weights, num_threads) + data.calc_paths_sum(py, selectors, node_offsets, weights, num_threads) } fn calc_paths_sum_impl( selectors: ArrayView1, - indices: ArrayView1, + node_offsets: ArrayView1, data: ArrayView2, weights: Option>, num_threads: usize, @@ -312,7 +339,7 @@ where { let mut paths = Array1::zeros(data.nrows()); - let indices = indices.as_slice().unwrap(); + let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); rayon::ThreadPoolBuilder::new() @@ -324,7 +351,7 @@ where .and(data.rows()) .par_for_each(|path, sample| { for (tree_start, tree_end) in - indices.iter().map(|i| *i as usize).tuple_windows() + node_offsets.iter().map(|i| *i as usize).tuple_windows() { let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; @@ -344,23 +371,30 @@ where } #[pyfunction] -#[pyo3(signature = (selectors, indices, data, leaf_count, weights = None, num_threads = 0))] +#[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, num_threads = 0))] pub(crate) fn calc_paths_sum_transpose<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, + leaf_offsets: Bound<'py, PyArray1>, data: Data<'py>, - leaf_count: usize, weights: Option>>, num_threads: usize, ) -> PyResult>> { - data.calc_paths_sum_transpose(py, selectors, indices, leaf_count, weights, num_threads) + data.calc_paths_sum_transpose( + py, + selectors, + node_offsets, + leaf_offsets, + weights, + num_threads, + ) } fn calc_paths_sum_transpose_impl( selectors: ArrayView1, - indices: ArrayView1, - leaf_count: usize, + node_offsets: ArrayView1, + leaf_offsets: ArrayView1, data: ArrayView2, weights: Option>, num_threads: usize, @@ -369,31 +403,40 @@ where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { - // We need leaf_offsets instead of leaf_counts here. - // It would allow to split the array and write safely from multiple threads. - let values = Arc::new((0..leaf_count).map(|_| Mutex::new(0.0)).collect::>()); - - let selectors = selectors.as_slice().unwrap(); + let selectors = selectors + .as_slice() + .expect("Cannot get selectors slice from ArrayView"); + let leaf_offsets = leaf_offsets + .as_slice() + .expect("Cannot get leaf_offsets slice from ArrayView"); + + let leaf_count = *leaf_offsets + .last() + .expect("leaf_offsets array cannot be empty") as usize; + let mut values = vec![0.0; leaf_count]; + let values_iter = mut_slices::MutSlices::new(&mut values, leaf_offsets); rayon::ThreadPoolBuilder::new() .num_threads(num_threads) .build() .expect("Cannot build rayon ThreadPool") .install(|| { - indices + node_offsets .iter() .map(|i| *i as usize) .tuple_windows() - .zip(iter::repeat_with(|| values.clone())) + .zip(values_iter) + .zip(leaf_offsets) .par_bridge() - .for_each(|((tree_start, tree_end), values)| { + .for_each(|(((tree_start, tree_end), values), &leaf_offset)| { for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - let mut value = values[leaf.left as usize].lock().unwrap(); + let value = + unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; if let Some(weights) = weights { *value += weights[x_index] * leaf.value; } else { @@ -403,28 +446,24 @@ where }) }); - Arc::try_unwrap(values) - .unwrap() - .into_iter() - .map(|mutex| mutex.into_inner().unwrap()) - .collect() + values.into() } #[pyfunction] -#[pyo3(signature = (selectors, indices, data, num_threads = 0))] +#[pyo3(signature = (selectors, node_offsets, data, num_threads = 0))] pub(crate) fn calc_feature_delta_sum<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, - indices: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, data: Data<'py>, num_threads: usize, ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { - data.calc_feature_delta_sum(py, selectors, indices, num_threads) + data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads) } fn calc_feature_delta_sum_impl( selectors: ArrayView1, - indices: ArrayView1, + node_offsets: ArrayView1, data: ArrayView2, num_threads: usize, ) -> (Array2, Array2) @@ -432,7 +471,7 @@ where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { - let indices = indices.as_slice().unwrap(); + let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); let mut delta_sum = Array2::zeros((data.nrows(), data.ncols())); @@ -448,7 +487,7 @@ where .and(hit_count.rows_mut()) .par_for_each(|sample, mut delta_sum_row, mut hit_count_row| { for (tree_start, tree_end) in - indices.iter().map(|i| *i as usize).tuple_windows() + node_offsets.iter().map(|i| *i as usize).tuple_windows() { let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index a42c4e2..22ae09d 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -1,226 +1,34 @@ -use rayon::iter::plumbing::{bridge, Consumer, Producer, ProducerCallback, UnindexedConsumer}; -use rayon::iter::{IndexedParallelIterator, ParallelIterator}; - pub struct MutSlices<'sl, 'off, T> { slice: &'sl mut [T], offsets: &'off [usize], + current: usize, } impl<'sl, 'off, T> MutSlices<'sl, 'off, T> { pub fn new(slice: &'sl mut [T], offsets: &'off [usize]) -> Self { - MutSlices { slice, offsets } + MutSlices { + slice, + offsets, + current: 0, + } } } -impl<'sl, T> Iterator for MutSlices<'sl, '_, T> { +impl<'sl, 'off, T> Iterator for MutSlices<'sl, 'off, T> { type Item = &'sl mut [T]; fn next(&mut self) -> Option { - if self.offsets.len() <= 1 { + if self.current >= self.offsets.len() - 1 { return None; } - // Split slice, save right back. - // take() temporarily replaces self.slice with an empty slice - let left: &mut [T]; - (left, self.slice) = - std::mem::take(&mut self.slice).split_at_mut(self.offsets[1] - self.offsets[0]); - - // Move offsets to the right - self.offsets = &self.offsets[1..]; + let start = self.offsets[self.current]; + let end = self.offsets[self.current + 1]; + self.current += 1; + // Here we temporarily replace slice with an empty one + let (left, right) = std::mem::take(&mut self.slice).split_at_mut(end - start); + self.slice = right; Some(left) } - - fn size_hint(&self) -> (usize, Option) { - let len = self.offsets.len() - 1; - (len, Some(len)) - } -} - -impl DoubleEndedIterator for MutSlices<'_, '_, T> { - fn next_back(&mut self) -> Option { - let offsets_len = self.offsets.len(); - if offsets_len <= 1 { - return None; - } - - // Split slice, save left back. - // take() temporarily replaces self.slice with an empty slice - let right: &mut [T]; - (self.slice, right) = std::mem::take(&mut self.slice) - .split_at_mut(self.offsets[offsets_len - 2] - self.offsets[0]); - - // Move offsets to the left - self.offsets = &self.offsets[..offsets_len - 1]; - - Some(right) - } -} - -impl ExactSizeIterator for MutSlices<'_, '_, T> { - fn len(&self) -> usize { - self.offsets.len() - 1 - } -} - -// Following rayon's ChunksMut implementation -impl<'sl, T> ParallelIterator for MutSlices<'sl, '_, T> -where - T: Send, -{ - type Item = &'sl mut [T]; - - fn drive_unindexed(self, consumer: C) -> C::Result - where - C: UnindexedConsumer, - { - bridge(self, consumer) - } - - fn opt_len(&self) -> Option { - ExactSizeIterator::len(self).into() - } -} - -impl IndexedParallelIterator for MutSlices<'_, '_, T> -where - T: Send, -{ - fn len(&self) -> usize { - ExactSizeIterator::len(self) - } - - fn drive(self, consumer: C) -> C::Result - where - C: Consumer, - { - bridge(self, consumer) - } - - fn with_producer(self, callback: CB) -> CB::Output - where - CB: ProducerCallback, - { - callback.callback(MutSlicesProducer { - slice: self.slice, - offsets: self.offsets, - }) - } -} - -struct MutSlicesProducer<'sl, 'off, T> { - slice: &'sl mut [T], - offsets: &'off [usize], -} - -impl<'sl, 'off, T> Producer for MutSlicesProducer<'sl, 'off, T> -where - T: Send, -{ - type Item = &'sl mut [T]; - type IntoIter = MutSlices<'sl, 'off, T>; - - fn into_iter(self) -> Self::IntoIter { - MutSlices::new(self.slice, self.offsets) - } - - fn split_at(self, index: usize) -> (Self, Self) { - let (left_slice, right_slice) = self - .slice - .split_at_mut(self.offsets[index] - self.offsets[0]); - let (left_offsets, right_offsets) = (&self.offsets[..=index], &self.offsets[index..]); - ( - MutSlicesProducer { - slice: left_slice, - offsets: left_offsets, - }, - MutSlicesProducer { - slice: right_slice, - offsets: right_offsets, - }, - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use itertools::Itertools; - - #[test] - fn test_mut_slices() { - let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let offsets = vec![0, 3, 6, 10]; - - let mut slices = MutSlices::new(&mut data, &offsets); - - assert_eq!(slices.next().unwrap(), &[1, 2, 3]); - assert_eq!(slices.next().unwrap(), &[4, 5, 6]); - assert_eq!(slices.next().unwrap(), &[7, 8, 9, 10]); - assert!(slices.next().is_none()); - - let mut slices = MutSlices::new(&mut data, &offsets); - - assert_eq!(slices.next_back().unwrap(), &[7, 8, 9, 10]); - assert_eq!(slices.next_back().unwrap(), &[4, 5, 6]); - assert_eq!(slices.next_back().unwrap(), &[1, 2, 3]); - assert!(slices.next_back().is_none()); - - let mut slices = MutSlices::new(&mut data, &offsets); - assert_eq!(slices.next().unwrap(), &[1, 2, 3]); - assert_eq!(slices.next_back().unwrap(), &[7, 8, 9, 10]); - assert_eq!(slices.next().unwrap(), &[4, 5, 6]); - assert_eq!(slices.next_back(), None); - assert_eq!(slices.next(), None); - } - - #[test] - fn test_mut_slices_len() { - let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let offsets = vec![0, 3, 6, 10]; - - let slices = MutSlices::new(&mut data, &offsets); - - assert_eq!(ExactSizeIterator::len(&slices), 3); - } - - #[test] - fn test_mut_slices_parallel() { - let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let offsets = vec![0, 3, 6, 10]; - - let slices = MutSlices::new(&mut data, &offsets); - - let sum: usize = ParallelIterator::map(slices, |slice| slice.iter().sum::()).sum(); - - assert_eq!(sum, data.iter().sum::()); - } - - #[test] - fn test_mut_slices_producer() { - let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let offsets = vec![0, 3, 6, 10]; - - let producer = MutSlicesProducer { - slice: &mut data, - offsets: &offsets, - }; - assert_eq!( - producer.into_iter().collect_vec(), - [vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9, 10]] - ); - - let producer = MutSlicesProducer { - slice: &mut data, - offsets: &offsets, - }; - let (left, right) = producer.split_at(1); - assert_eq!(left.into_iter().collect_vec(), [&[1, 2, 3]]); - assert_eq!( - right.into_iter().collect_vec(), - [&vec![4, 5, 6], &vec![7, 8, 9, 10]] - ); - } } diff --git a/src/coniferest/aadforest.py b/src/coniferest/aadforest.py index f276bc4..9e88973 100644 --- a/src/coniferest/aadforest.py +++ b/src/coniferest/aadforest.py @@ -37,14 +37,7 @@ def score_samples(self, x, weights=None): if weights is None: weights = self.weights - return calc_paths_sum( - self.selectors, - self.node_offsets, - x, - weights, - num_threads=self.num_threads, - batch_size=self.get_batch_size(self.n_trees), - ) + return calc_paths_sum(self.selectors, self.node_offsets, x, weights, num_threads=self.num_threads) def loss( self, @@ -192,12 +185,7 @@ def __init__( map_value=None, ): super().__init__( - trees=[], - n_subsamples=n_subsamples, - max_depth=max_depth, - n_jobs=n_jobs, - random_seed=random_seed, - sampletrees_per_batch=sampletrees_per_batch, + trees=[], n_subsamples=n_subsamples, max_depth=max_depth, n_jobs=n_jobs, random_seed=random_seed ) self.n_trees = n_trees diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index 5bc2444..d89c107 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -11,7 +11,7 @@ class ForestEvaluator: selector_dtype = selector_dtype - def __init__(self, samples, selectors, indices, leaf_count, *, num_threads): + def __init__(self, samples, selectors, node_offsets, leaf_offsets, *, num_threads): """ Base class for the forest evaluators. Does the trivial job: * runs calc_paths_sum written in Rust, @@ -42,9 +42,10 @@ def __init__(self, samples, selectors, indices, leaf_count, *, num_threads): self.node_offsets = node_offsets self.leaf_offsets = leaf_offsets - if num_threads is None or num_threads < 0: - # Ask Rust's rayon to use all available threads - self.num_threads = 0 + if num_threads is None or num_threads < 1: + # Count of available CPUs is not a simple thing, see loky's implementation here: + # https://github.com/joblib/joblib/blob/476ff8e62b221fc5816bad9b55dec8883d4f157c/joblib/externals/loky/backend/context.py#L83 + self.num_threads = joblib.cpu_count() else: self.num_threads = num_threads @@ -103,7 +104,7 @@ def combine_selectors(cls, selectors_list): node_offsets[1:] = np.add.accumulate(lens) for i in range(len(selectors_list)): - selectors[indices[i]: indices[i + 1]] = selectors_list[i] + selectors[node_offsets[i]: node_offsets[i + 1]] = selectors_list[i] # Assign a unique sequential index to every leaf # The index is used for weighted scores @@ -145,8 +146,8 @@ def score_samples(self, x): return -( 2 ** ( - -calc_paths_sum(self.selectors, self.indices, x, num_threads=self.num_threads) - / (self.average_path_length(self.samples) * trees) + -calc_paths_sum(self.selectors, self.node_offsets, x, num_threads=self.num_threads) + / (self.average_path_length(self.samples) * self.n_trees) ) ) @@ -154,13 +155,7 @@ def _feature_delta_sum(self, x): if not x.flags["C_CONTIGUOUS"]: x = np.ascontiguousarray(x) - return calc_feature_delta_sum( - self.selectors, - self.node_offsets, - x, - num_threads=self.num_threads, - batch_size=self.get_batch_size(self.n_trees), - ) + return calc_feature_delta_sum(self.selectors, self.node_offsets, x, num_threads=self.num_threads) def feature_signature(self, x): delta_sum, hit_count = self._feature_delta_sum(x) From 71ec7beaa9710cf3dfbca864f002d4ce97dd2df5 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 15 May 2024 20:16:09 -0300 Subject: [PATCH 06/40] Make ABI3 optional --- rust/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 78824da..fa2378f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -11,10 +11,13 @@ crate-type = ["cdylib"] [profile.dev] opt-level = 3 +[features] +default = ["pyo3/abi3-py39"] + [dependencies] enum_dispatch = "0.3" itertools = "0.12" -pyo3 = { version = "0.21", features = ["abi3-py39", "extension-module"] } +pyo3 = { version = "0.21", features = ["extension-module"] } # Needs to be consistent with ndarray dependecy in numpy ndarray = { version = "0.15", features = ["rayon"] } num-traits = "0.2" From b5d4890f1e17071dc0e67554de54b7f63df8edaa Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 21 May 2024 12:37:07 -0400 Subject: [PATCH 07/40] Make selector's dtype a OnceCell --- rust/src/lib.rs | 73 +++----------------------------------------- rust/src/selector.rs | 8 ++--- 2 files changed, 7 insertions(+), 74 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index b5095a9..4bab931 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,82 +1,19 @@ mod mut_slices; +mod selector; +use crate::mut_slices::MutSlices; +use crate::selector::Selector; use enum_dispatch::enum_dispatch; use itertools::Itertools; use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis, Zip}; use num_traits::AsPrimitive; use numpy::PyArrayMethods; -use numpy::{Element, PyArray, PyArrayDescr}; +use numpy::{Element, PyArray}; use numpy::{PyArray1, PyArray2}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::py_run; -use pyo3::types::PyDict; use rayon::prelude::*; -/// Selector is the representation of decision tree nodes: either branches or leafs. -/// -/// We use "C"-representation with standard alignment (np.dtype(align=True)), but "packed" -/// (dtype(aligh=False)) would work as well. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub(crate) struct Selector { - /// Feature index to branch on, -1.0 if leaf - feature: i32, - /// Index of left subtree, leaf_id if leaf - left: i32, - /// Feature value to branch on, resulting decision score if leaf - value: f64, - /// Index of right subtree, -1 if leaf - right: i32, - /// Natural logarithm of the number of samples in the node - log_n_node_samples: f32, -} - -impl Selector { - pub(crate) fn dtype(py: Python) -> PyResult> { - let locals = PyDict::new_bound(py); - py_run!( - py, - *locals, - r#" - dtype = __import__('numpy').dtype( - [ - ('feature', 'i4'), - ('left', 'i4'), - ('value', 'f8'), - ('right', 'i4'), - ('log_n_node_samples', 'f4') - ], - align=True, - ) - "# - ); - Ok(locals - .get_item("dtype") - .expect("Error in built-in Python code for dtype initialization") - .expect("Error in built-in Python code for dtype initialization: dtype cannot be None") - .downcast::()? - .clone()) - } - - #[inline(always)] - pub(crate) fn is_leaf(&self) -> bool { - self.feature == -1 - } -} - -/// Implementation of [numpy::Element] for [Selector] -/// -/// Safety: we guarantee that [Selector] has the same layout as it would have in numpy with -/// [Selector::dtype] -unsafe impl Element for Selector { - const IS_COPY: bool = true; - - fn get_dtype_bound(py: Python) -> Bound { - Self::dtype(py).unwrap() - } -} - #[enum_dispatch] trait DataTrait<'py> { fn calc_paths_sum( @@ -414,7 +351,7 @@ where .last() .expect("leaf_offsets array cannot be empty") as usize; let mut values = vec![0.0; leaf_count]; - let values_iter = mut_slices::MutSlices::new(&mut values, leaf_offsets); + let values_iter = MutSlices::new(&mut values, leaf_offsets); rayon::ThreadPoolBuilder::new() .num_threads(num_threads) diff --git a/rust/src/selector.rs b/rust/src/selector.rs index 498f12d..850a909 100644 --- a/rust/src/selector.rs +++ b/rust/src/selector.rs @@ -29,7 +29,7 @@ impl Selector { pub(crate) fn dtype(py: Python) -> PyResult> { let unbind_dtype = SELECTOR_DTYPE_CELL.get_or_try_init(py, || -> PyResult<_> { - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); py_run!( py, *locals, @@ -69,11 +69,7 @@ impl Selector { unsafe impl Element for Selector { const IS_COPY: bool = true; - fn get_dtype(py: Python) -> Bound { + fn get_dtype_bound(py: Python) -> Bound { Self::dtype(py).unwrap() } - - fn clone_ref(&self, _py: Python<'_>) -> Self { - *self - } } From 4d3ccb7f0b17d9b228e93f57691c6c7d398dfb5e Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 21 May 2024 12:55:11 -0400 Subject: [PATCH 08/40] More rust modules --- rust/src/lib.rs | 464 +------------------------------ rust/src/tree_traversal.rs | 545 +++++++++++-------------------------- 2 files changed, 166 insertions(+), 843 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 4bab931..4957bd8 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,471 +1,15 @@ mod mut_slices; mod selector; +mod tree_traversal; -use crate::mut_slices::MutSlices; use crate::selector::Selector; -use enum_dispatch::enum_dispatch; -use itertools::Itertools; -use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis, Zip}; -use num_traits::AsPrimitive; -use numpy::PyArrayMethods; -use numpy::{Element, PyArray}; -use numpy::{PyArray1, PyArray2}; -use pyo3::exceptions::PyValueError; +use crate::tree_traversal::{calc_feature_delta_sum, calc_paths_sum, calc_paths_sum_transpose}; use pyo3::prelude::*; -use rayon::prelude::*; - -#[enum_dispatch] -trait DataTrait<'py> { - fn calc_paths_sum( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - weights: Option>>, - num_threads: usize, - ) -> PyResult>>; - - fn calc_paths_sum_transpose( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - leaf_offsets: Bound<'py, PyArray1>, - weights: Option>>, - num_threads: usize, - ) -> PyResult>>; - - fn calc_feature_delta_sum( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - num_threads: usize, - ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)>; -} - -impl<'py, T> DataTrait<'py> for Bound<'py, PyArray2> -where - T: Element + Copy + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - fn calc_paths_sum( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - weights: Option>>, - num_threads: usize, - ) -> PyResult>> { - let selectors = selectors.readonly(); - let selectors_view = selectors.as_array(); - check_selectors(selectors_view)?; - - let node_offsets = node_offsets.readonly(); - let node_offsets_view = node_offsets.as_array(); - check_node_offsets(node_offsets_view, selectors.len()?)?; - - let data = self.readonly(); - let data_view = data.as_array(); - check_data(data_view)?; - - let weights = weights.map(|weights| weights.readonly()); - let weights_view = weights.as_ref().map(|weights| weights.as_array()); - - // Here we need to dispatch `data` and run the template function - let values = calc_paths_sum_impl( - selectors_view, - node_offsets_view, - data_view, - weights_view, - num_threads, - ); - Ok(PyArray::from_owned_array_bound(py, values)) - } - - fn calc_paths_sum_transpose( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - leaf_offsets: Bound<'py, PyArray1>, - weights: Option>>, - num_threads: usize, - ) -> PyResult>> { - let selectors = selectors.readonly(); - let selectors_view = selectors.as_array(); - check_selectors(selectors_view)?; - - let node_offsets = node_offsets.readonly(); - let node_offsets_view = node_offsets.as_array(); - check_node_offsets(node_offsets_view, selectors_view.len())?; - - let leaf_offsets = leaf_offsets.readonly(); - let leaf_offsets_view = leaf_offsets.as_array(); - check_leaf_offsets(leaf_offsets_view, node_offsets_view.len())?; - - let data = self.readonly(); - let data_view = data.as_array(); - check_data(data_view)?; - - let weights = weights.map(|weights| weights.readonly()); - let weights_view = weights.as_ref().map(|weights| weights.as_array()); - - // Here we need to dispatch `data` and run the template function - let values = calc_paths_sum_transpose_impl( - selectors_view, - node_offsets_view, - leaf_offsets_view, - data_view, - weights_view, - num_threads, - ); - Ok(PyArray::from_owned_array_bound(py, values)) - } - - fn calc_feature_delta_sum( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - num_threads: usize, - ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { - let selectors = selectors.readonly(); - let selectors_view = selectors.as_array(); - check_selectors(selectors_view)?; - - let node_offsets = node_offsets.readonly(); - let node_offsets_view = node_offsets.as_array(); - check_node_offsets(node_offsets_view, selectors.len()?)?; - - let data = self.readonly(); - let data_view = data.as_array(); - check_data(data_view)?; - - let (delta_sum, hit_count) = - calc_feature_delta_sum_impl(selectors_view, node_offsets_view, data_view, num_threads); - - let delta_sum = PyArray::from_owned_array_bound(py, delta_sum); - let hit_count = PyArray::from_owned_array_bound(py, hit_count); - - Ok((delta_sum, hit_count)) - } -} - -#[enum_dispatch(DataTrait)] -#[derive(FromPyObject)] -enum Data<'py> { - F64(Bound<'py, PyArray2>), - F32(Bound<'py, PyArray2>), -} - -// It looks like the performance is not affected by returning a copy of Selector, not reference. -#[inline] -fn find_leaf(tree: &[Selector], sample: &[T]) -> Selector -where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let mut i = 0; - loop { - let selector = *unsafe { tree.get_unchecked(i) }; - if selector.is_leaf() { - break selector; - } - - // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? - let threshold: T = selector.value.as_(); - i = if *unsafe { sample.get_unchecked(selector.feature as usize) } <= threshold { - selector.left as usize - } else { - selector.right as usize - }; - } -} - -#[inline] -fn check_selectors(selectors: ArrayView1) -> PyResult<()> { - if !selectors.is_standard_layout() { - return Err(PyValueError::new_err( - "selectors must be contiguous and in memory order", - )); - } - Ok(()) -} - -#[inline] -fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) -> PyResult<()> { - if let Some(node_offsets) = node_offsets.as_slice() { - for (x, y) in node_offsets.iter().copied().tuple_windows() { - if x > y { - return Err(PyValueError::new_err( - "node_offsets must be sorted in ascending order", - )); - } - } - if node_offsets[node_offsets.len() - 1] as usize > selectors_length { - return Err(PyValueError::new_err( - "node_offsets are out of range of the selectors", - )); - } - Ok(()) - } else { - Err(PyValueError::new_err( - "node_offsets must be contiguous and in memory order", - )) - } -} - -#[inline] -fn check_leaf_offsets(leaf_offsets: ArrayView1, node_offset_len: usize) -> PyResult<()> { - if leaf_offsets.len() != node_offset_len { - return Err(PyValueError::new_err( - "leaf_offsets must have the same length as node_offsets", - )); - } - if let Some(leaf_offsets) = leaf_offsets.as_slice() { - for (x, y) in leaf_offsets.iter().copied().tuple_windows() { - if x > y { - return Err(PyValueError::new_err( - "leaf_offsets must be sorted in ascending order", - )); - } - } - Ok(()) - } else { - Err(PyValueError::new_err( - "leaf_offsets must be contiguous and in memory order", - )) - } -} - -#[inline] -fn check_data(data: ArrayView2) -> PyResult<()> { - if !data.is_standard_layout() { - return Err(PyValueError::new_err( - "data must be contiguous and in memory order", - )); - } - Ok(()) -} - -#[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, weights = None, num_threads = 0))] -pub(crate) fn calc_paths_sum<'py>( - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - // TODO: support f32 data - data: Data<'py>, - weights: Option>>, - num_threads: usize, -) -> PyResult>> { - data.calc_paths_sum(py, selectors, node_offsets, weights, num_threads) -} - -fn calc_paths_sum_impl( - selectors: ArrayView1, - node_offsets: ArrayView1, - data: ArrayView2, - weights: Option>, - num_threads: usize, -) -> Array1 -where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let mut paths = Array1::zeros(data.nrows()); - - let node_offsets = node_offsets.as_slice().unwrap(); - let selectors = selectors.as_slice().unwrap(); - - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - Zip::from(paths.view_mut()) - .and(data.rows()) - .par_for_each(|path, sample| { - for (tree_start, tree_end) in - node_offsets.iter().map(|i| *i as usize).tuple_windows() - { - let tree_selectors = - unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - if let Some(weights) = weights { - *path += *unsafe { weights.uget(leaf.left as usize) } * leaf.value; - } else { - *path += leaf.value; - } - } - }) - }); - - paths -} - -#[pyfunction] -#[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, num_threads = 0))] -pub(crate) fn calc_paths_sum_transpose<'py>( - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - leaf_offsets: Bound<'py, PyArray1>, - data: Data<'py>, - weights: Option>>, - num_threads: usize, -) -> PyResult>> { - data.calc_paths_sum_transpose( - py, - selectors, - node_offsets, - leaf_offsets, - weights, - num_threads, - ) -} - -fn calc_paths_sum_transpose_impl( - selectors: ArrayView1, - node_offsets: ArrayView1, - leaf_offsets: ArrayView1, - data: ArrayView2, - weights: Option>, - num_threads: usize, -) -> Array1 -where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let selectors = selectors - .as_slice() - .expect("Cannot get selectors slice from ArrayView"); - let leaf_offsets = leaf_offsets - .as_slice() - .expect("Cannot get leaf_offsets slice from ArrayView"); - - let leaf_count = *leaf_offsets - .last() - .expect("leaf_offsets array cannot be empty") as usize; - let mut values = vec![0.0; leaf_count]; - let values_iter = MutSlices::new(&mut values, leaf_offsets); - - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - node_offsets - .iter() - .map(|i| *i as usize) - .tuple_windows() - .zip(values_iter) - .zip(leaf_offsets) - .par_bridge() - .for_each(|(((tree_start, tree_end), values), &leaf_offset)| { - for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { - let tree_selectors = - unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - let value = - unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; - if let Some(weights) = weights { - *value += weights[x_index] * leaf.value; - } else { - *value += leaf.value; - } - } - }) - }); - - values.into() -} - -#[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, num_threads = 0))] -pub(crate) fn calc_feature_delta_sum<'py>( - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - data: Data<'py>, - num_threads: usize, -) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { - data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads) -} - -fn calc_feature_delta_sum_impl( - selectors: ArrayView1, - node_offsets: ArrayView1, - data: ArrayView2, - num_threads: usize, -) -> (Array2, Array2) -where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let node_offsets = node_offsets.as_slice().unwrap(); - let selectors = selectors.as_slice().unwrap(); - - let mut delta_sum = Array2::zeros((data.nrows(), data.ncols())); - let mut hit_count = Array2::zeros((data.nrows(), data.ncols())); - - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - Zip::from(data.rows()) - .and(delta_sum.rows_mut()) - .and(hit_count.rows_mut()) - .par_for_each(|sample, mut delta_sum_row, mut hit_count_row| { - for (tree_start, tree_end) in - node_offsets.iter().map(|i| *i as usize).tuple_windows() - { - let tree_selectors = - unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let mut i = 0; - let mut parent_selector: &Selector; - loop { - parent_selector = unsafe { tree_selectors.get_unchecked(i) }; - if parent_selector.is_leaf() { - break; - } - - // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? - let threshold: T = parent_selector.value.as_(); - i = if *unsafe { sample.uget(parent_selector.feature as usize) } - <= threshold - { - parent_selector.left as usize - } else { - parent_selector.right as usize - }; - - let child_selector = unsafe { tree_selectors.get_unchecked(i) }; - *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += - 1.0 + 2.0 - * (child_selector.log_n_node_samples as f64 - - parent_selector.log_n_node_samples as f64); - *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += - 1; - } - } - }); - }); - - (delta_sum, hit_count) -} #[pymodule] #[pyo3(name = "calc_paths_sum")] -fn rust_module(_py: Python, m: &Bound) -> PyResult<()> { - m.add("selector_dtype", Selector::dtype(_py)?)?; +fn rust_module(py: Python, m: &Bound) -> PyResult<()> { + m.add("selector_dtype", Selector::dtype(py)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum, m)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum_transpose, m)?)?; m.add_function(wrap_pyfunction!(calc_feature_delta_sum, m)?)?; diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 2180d24..3d76d0b 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -2,16 +2,13 @@ use crate::mut_slices::MutSlices; use crate::selector::Selector; use enum_dispatch::enum_dispatch; use itertools::Itertools; -use ndarray::parallel::prelude::*; -use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Axis, Zip}; +use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis, Zip}; use num_traits::AsPrimitive; -use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; +use numpy::{Element, PyArray, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; -use rayon::prelude::*; - -type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); +use rayon::iter::{ParallelBridge, ParallelIterator}; #[enum_dispatch] trait DataTrait<'py> { @@ -22,10 +19,8 @@ trait DataTrait<'py> { node_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, - batch_size: usize, ) -> PyResult>>; - #[allow(clippy::too_many_arguments)] fn calc_paths_sum_transpose( &self, py: Python<'py>, @@ -34,7 +29,6 @@ trait DataTrait<'py> { leaf_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, - batch_size: usize, ) -> PyResult>>; fn calc_feature_delta_sum( @@ -43,17 +37,7 @@ trait DataTrait<'py> { selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, num_threads: usize, - batch_size: usize, - ) -> PyResult>; - - fn calc_apply( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - num_threads: usize, - batch_size: usize, - ) -> PyResult>>; + ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)>; } impl<'py, T> DataTrait<'py> for Bound<'py, PyArray2> @@ -68,7 +52,6 @@ where node_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, - batch_size: usize, ) -> PyResult>> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); @@ -85,25 +68,15 @@ where let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); - let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; - - Ok({ - let paths = PyArray1::zeros(py, data_view.nrows(), false); - // SAFETY: this call invalidates other views, but it is the only view we need - let paths_view_mut = unsafe { paths.as_array_mut() }; - - // Here we need to dispatch `data` and run the template function - calc_paths_sum_impl( - selectors_view, - node_offsets_view, - data_view, - weights_view, - num_threads, - batch_size, - paths_view_mut, - ); - paths - }) + // Here we need to dispatch `data` and run the template function + let values = calc_paths_sum_impl( + selectors_view, + node_offsets_view, + data_view, + weights_view, + num_threads, + ); + Ok(PyArray::from_owned_array_bound(py, values)) } fn calc_paths_sum_transpose( @@ -114,7 +87,6 @@ where leaf_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, - batch_size: usize, ) -> PyResult>> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); @@ -135,33 +107,16 @@ where let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); - let num_threads = get_num_threads(data_view.ncols(), num_threads, batch_size)?; - - Ok({ - let values = PyArray1::zeros( - py, - *leaf_offsets_view - .last() - .expect("leaf_offsets array must not be empty"), - false, - ); - - // SAFETY: this call invalidates other views, but it is the only view we need - let values_view = unsafe { values.as_array_mut() }; - - // Here we need to dispatch `data` and run the template function - calc_paths_sum_transpose_impl( - selectors_view, - node_offsets_view, - leaf_offsets_view, - data_view, - weights_view, - num_threads, - batch_size, - values_view, - ); - values - }) + // Here we need to dispatch `data` and run the template function + let values = calc_paths_sum_transpose_impl( + selectors_view, + node_offsets_view, + leaf_offsets_view, + data_view, + weights_view, + num_threads, + ); + Ok(PyArray::from_owned_array_bound(py, values)) } fn calc_feature_delta_sum( @@ -170,8 +125,7 @@ where selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, num_threads: usize, - batch_size: usize, - ) -> PyResult> { + ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); check_selectors(selectors_view)?; @@ -184,70 +138,13 @@ where let data_view = data.as_array(); check_data(data_view)?; - let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; - - Ok({ - let delta_sum = PyArray2::zeros(py, (data_view.nrows(), data_view.ncols()), false); - let hit_count = PyArray2::zeros(py, (data_view.nrows(), data_view.ncols()), false); - - // SAFETY: this call invalidates other views, but it is the only view we need - let delta_sum_view = unsafe { delta_sum.as_array_mut() }; - // SAFETY: this call invalidates other views, but it is the only view we need - let hit_count_view = unsafe { hit_count.as_array_mut() }; - - calc_feature_delta_sum_impl( - selectors_view, - node_offsets_view, - data_view, - num_threads, - batch_size, - delta_sum_view, - hit_count_view, - ); - - (delta_sum, hit_count) - }) - } + let (delta_sum, hit_count) = + calc_feature_delta_sum_impl(selectors_view, node_offsets_view, data_view, num_threads); - fn calc_apply( - &self, - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - num_threads: usize, - batch_size: usize, - ) -> PyResult>> { - let selectors = selectors.readonly(); - let selectors_view = selectors.as_array(); - check_selectors(selectors_view)?; - - let node_offsets = node_offsets.readonly(); - let node_offsets_view = node_offsets.as_array(); - check_node_offsets(node_offsets_view, selectors.len()?)?; - - let data = self.readonly(); - let data_view = data.as_array(); - check_data(data_view)?; + let delta_sum = PyArray::from_owned_array_bound(py, delta_sum); + let hit_count = PyArray::from_owned_array_bound(py, hit_count); - let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; - - Ok({ - let leafs = - PyArray2::zeros(py, (data_view.nrows(), node_offsets_view.len() - 1), false); - // SAFETY: this call invalidates other views, but it is the only view we need - let leafs_view = unsafe { leafs.as_array_mut() }; - - calc_apply_impl( - selectors_view, - node_offsets_view, - data_view, - num_threads, - batch_size, - leafs_view, - ); - - leafs - }) + Ok((delta_sum, hit_count)) } } @@ -294,11 +191,6 @@ fn check_selectors(selectors: ArrayView1) -> PyResult<()> { #[inline] fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) -> PyResult<()> { - if node_offsets.len() <= 1 { - return Err(PyValueError::new_err( - "node_offsets must have at least two elements", - )); - } if let Some(node_offsets) = node_offsets.as_slice() { for (x, y) in node_offsets.iter().copied().tuple_windows() { if x > y { @@ -307,13 +199,12 @@ fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) )); } } - if node_offsets[node_offsets.len() - 1] > selectors_length { - Err(PyValueError::new_err( + if node_offsets[node_offsets.len() - 1] as usize > selectors_length { + return Err(PyValueError::new_err( "node_offsets are out of range of the selectors", - )) - } else { - Ok(()) + )); } + Ok(()) } else { Err(PyValueError::new_err( "node_offsets must be contiguous and in memory order", @@ -323,11 +214,6 @@ fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) #[inline] fn check_leaf_offsets(leaf_offsets: ArrayView1, node_offset_len: usize) -> PyResult<()> { - if leaf_offsets.len() <= 1 { - return Err(PyValueError::new_err( - "leaf_offsets must have at least two elements", - )); - } if leaf_offsets.len() != node_offset_len { return Err(PyValueError::new_err( "leaf_offsets must have the same length as node_offsets", @@ -359,18 +245,8 @@ fn check_data(data: ArrayView2) -> PyResult<()> { Ok(()) } -#[inline] -fn get_num_threads(nrows: usize, num_threads: usize, batch_size: usize) -> PyResult { - if batch_size == 0 { - Err(PyValueError::new_err("batch_size must be greater than 0")) - } else { - let n_jobs = nrows.div_ceil(batch_size); - Ok(usize::min(num_threads, n_jobs)) - } -} - #[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, weights = None, *, num_threads, batch_size))] +#[pyo3(signature = (selectors, node_offsets, data, weights = None, num_threads = 0))] pub(crate) fn calc_paths_sum<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, @@ -379,16 +255,8 @@ pub(crate) fn calc_paths_sum<'py>( data: Data<'py>, weights: Option>>, num_threads: usize, - batch_size: usize, ) -> PyResult>> { - data.calc_paths_sum( - py, - selectors, - node_offsets, - weights, - num_threads, - batch_size, - ) + data.calc_paths_sum(py, selectors, node_offsets, weights, num_threads) } fn calc_paths_sum_impl( @@ -397,49 +265,46 @@ fn calc_paths_sum_impl( data: ArrayView2, weights: Option>, num_threads: usize, - batch_size: usize, - paths: ArrayViewMut1, -) where +) -> Array1 +where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { + let mut paths = Array1::zeros(data.nrows()); + let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); - let inner_fn = |path: &mut f64, sample: ArrayView1| { - for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { - let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - if let Some(weights) = weights { - *path += *unsafe { weights.uget(leaf.left as usize) } * leaf.value; - } else { - *path += leaf.value; - } - } - }; - - let zip = Zip::from(paths).and(data.rows()); - - if num_threads == 1 { - zip.for_each(inner_fn); - } else { - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - zip.into_par_iter() - .with_min_len(batch_size) - .for_each(|(path, sample)| inner_fn(path, sample)); - }); - } + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + Zip::from(paths.view_mut()) + .and(data.rows()) + .par_for_each(|path, sample| { + for (tree_start, tree_end) in + node_offsets.iter().map(|i| *i as usize).tuple_windows() + { + let tree_selectors = + unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + if let Some(weights) = weights { + *path += *unsafe { weights.uget(leaf.left as usize) } * leaf.value; + } else { + *path += leaf.value; + } + } + }) + }); + + paths } -#[allow(clippy::too_many_arguments)] #[pyfunction] -#[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, *, num_threads, batch_size))] +#[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, num_threads = 0))] pub(crate) fn calc_paths_sum_transpose<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, @@ -448,7 +313,6 @@ pub(crate) fn calc_paths_sum_transpose<'py>( data: Data<'py>, weights: Option>>, num_threads: usize, - batch_size: usize, ) -> PyResult>> { data.calc_paths_sum_transpose( py, @@ -457,11 +321,9 @@ pub(crate) fn calc_paths_sum_transpose<'py>( leaf_offsets, weights, num_threads, - batch_size, ) } -#[allow(clippy::too_many_arguments)] fn calc_paths_sum_transpose_impl( selectors: ArrayView1, node_offsets: ArrayView1, @@ -469,85 +331,67 @@ fn calc_paths_sum_transpose_impl( data: ArrayView2, weights: Option>, num_threads: usize, - batch_size: usize, - mut values: ArrayViewMut1, -) where +) -> Array1 +where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { let selectors = selectors .as_slice() - .expect("selectors must be contiguous and in memory order"); - let node_offsets = node_offsets - .as_slice() - .expect("node_offsets must be contiguous and in memory order"); + .expect("Cannot get selectors slice from ArrayView"); let leaf_offsets = leaf_offsets .as_slice() - .expect("leaf_offsets must be contiguous and in memory order"); - - let values_iter = MutSlices::new( - values - .as_slice_mut() - .expect("values must be contiguous and in memory order"), - leaf_offsets, - ); - - let inner_fn = - |(((tree_start, tree_end), values), &leaf_first): (((usize, usize), &mut [f64]), _)| { - for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { - let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - let value = unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_first) }; - if let Some(weights) = weights { - *value += weights[x_index] * leaf.value; - } else { - *value += leaf.value; - } - } - }; - - let leaf_firsts = &leaf_offsets[..leaf_offsets.len() - 1]; - - if num_threads == 1 { - // Here we use itertools methods - node_offsets - .iter() - .copied() - .tuple_windows() - .zip_eq(values_iter) - .zip_eq(leaf_firsts) - .for_each(inner_fn); - } else { - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - // Here we use rayon methods - node_offsets - .par_windows(2) - .map(|window| (window[0], window[1])) - .zip_eq(values_iter) - .zip_eq(leaf_firsts) - .with_min_len(batch_size) - .for_each(inner_fn); - }); - } + .expect("Cannot get leaf_offsets slice from ArrayView"); + + let leaf_count = *leaf_offsets + .last() + .expect("leaf_offsets array cannot be empty") as usize; + let mut values = vec![0.0; leaf_count]; + let values_iter = MutSlices::new(&mut values, leaf_offsets); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + node_offsets + .iter() + .map(|i| *i as usize) + .tuple_windows() + .zip(values_iter) + .zip(leaf_offsets) + .par_bridge() + .for_each(|(((tree_start, tree_end), values), &leaf_offset)| { + for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { + let tree_selectors = + unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + let value = + unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; + if let Some(weights) = weights { + *value += weights[x_index] * leaf.value; + } else { + *value += leaf.value; + } + } + }) + }); + + values.into() } #[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, *, num_threads, batch_size))] +#[pyo3(signature = (selectors, node_offsets, data, num_threads = 0))] pub(crate) fn calc_feature_delta_sum<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, data: Data<'py>, num_threads: usize, - batch_size: usize, -) -> PyResult> { - data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads, batch_size) +) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { + data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads) } fn calc_feature_delta_sum_impl( @@ -555,126 +399,61 @@ fn calc_feature_delta_sum_impl( node_offsets: ArrayView1, data: ArrayView2, num_threads: usize, - batch_size: usize, - mut delta_sum: ArrayViewMut2, - mut hit_count: ArrayViewMut2, -) where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let node_offsets = node_offsets.as_slice().unwrap(); - let selectors = selectors.as_slice().unwrap(); - - let inner_fn = |sample: ArrayView1, - mut delta_sum_row: ArrayViewMut1, - mut hit_count_row: ArrayViewMut1| { - for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { - let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let mut i = 0; - let mut parent_selector: &Selector; - loop { - parent_selector = unsafe { tree_selectors.get_unchecked(i) }; - if parent_selector.is_leaf() { - break; - } - - // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? - let threshold: T = parent_selector.value.as_(); - i = if *unsafe { sample.uget(parent_selector.feature as usize) } <= threshold { - parent_selector.left as usize - } else { - parent_selector.right as usize - }; - - let child_selector = unsafe { tree_selectors.get_unchecked(i) }; - // Here we cast to f64 following the original Cython implementation, but - // it is a subject to change. - *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += 1.0 - + 2.0 - * (child_selector.log_n_node_samples - parent_selector.log_n_node_samples) - as f64; - *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += 1; - } - } - }; - - let zip = Zip::from(data.rows()) - .and(delta_sum.rows_mut()) - .and(hit_count.rows_mut()); - - if num_threads == 1 { - zip.for_each(inner_fn); - } else { - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - zip.into_par_iter().with_min_len(batch_size).for_each( - |(sample, delta_sum_row, hit_count_row)| { - inner_fn(sample, delta_sum_row, hit_count_row) - }, - ); - }); - } -} - -#[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, *, num_threads, batch_size))] -pub(crate) fn calc_apply<'py>( - py: Python<'py>, - selectors: Bound<'py, PyArray1>, - node_offsets: Bound<'py, PyArray1>, - data: Data<'py>, - num_threads: usize, - batch_size: usize, -) -> PyResult>> { - data.calc_apply(py, selectors, node_offsets, num_threads, batch_size) -} - -fn calc_apply_impl( - selectors: ArrayView1, - node_offsets: ArrayView1, - data: ArrayView2, - num_threads: usize, - batch_size: usize, - mut leafs: ArrayViewMut2, -) where +) -> (Array2, Array2) +where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); - let inner_fn = |sample: ArrayView1, mut sample_leafs: ArrayViewMut1| { - let sample_slice = sample.as_slice().unwrap(); - let leafs_slice = sample_leafs.as_slice_mut().unwrap(); - for ((tree_start, tree_end), leaf_id) in node_offsets - .iter() - .copied() - .tuple_windows() - .zip(leafs_slice.iter_mut()) - { - let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; - let leaf = find_leaf(tree_selectors, sample_slice); - *leaf_id = leaf.left; - } - }; - - let zip = Zip::from(data.rows()).and(leafs.rows_mut()); - - if num_threads == 1 { - zip.for_each(inner_fn); - } else { - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - zip.into_par_iter() - .with_min_len(batch_size) - .for_each(|(sample, sample_leafs)| inner_fn(sample, sample_leafs)); - }); - } + let mut delta_sum = Array2::zeros((data.nrows(), data.ncols())); + let mut hit_count = Array2::zeros((data.nrows(), data.ncols())); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + Zip::from(data.rows()) + .and(delta_sum.rows_mut()) + .and(hit_count.rows_mut()) + .par_for_each(|sample, mut delta_sum_row, mut hit_count_row| { + for (tree_start, tree_end) in + node_offsets.iter().map(|i| *i as usize).tuple_windows() + { + let tree_selectors = + unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let mut i = 0; + let mut parent_selector: &Selector; + loop { + parent_selector = unsafe { tree_selectors.get_unchecked(i) }; + if parent_selector.is_leaf() { + break; + } + + // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? + let threshold: T = parent_selector.value.as_(); + i = if *unsafe { sample.uget(parent_selector.feature as usize) } + <= threshold + { + parent_selector.left as usize + } else { + parent_selector.right as usize + }; + + let child_selector = unsafe { tree_selectors.get_unchecked(i) }; + *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += + 1.0 + 2.0 + * (child_selector.log_n_node_samples as f64 + - parent_selector.log_n_node_samples as f64); + *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += + 1; + } + } + }); + }); + + (delta_sum, hit_count) } From 3667a7992746c61ede0dec66f5583365812dbc74 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 21 May 2024 13:03:11 -0400 Subject: [PATCH 09/40] Fix clippy lints --- rust/src/tree_traversal.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 3d76d0b..e127ec3 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -10,6 +10,8 @@ use pyo3::prelude::PyAnyMethods; use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; use rayon::iter::{ParallelBridge, ParallelIterator}; +type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); + #[enum_dispatch] trait DataTrait<'py> { fn calc_paths_sum( @@ -37,7 +39,7 @@ trait DataTrait<'py> { selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, num_threads: usize, - ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)>; + ) -> PyResult>; } impl<'py, T> DataTrait<'py> for Bound<'py, PyArray2> @@ -125,7 +127,7 @@ where selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, num_threads: usize, - ) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { + ) -> PyResult> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); check_selectors(selectors_view)?; @@ -199,7 +201,7 @@ fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) )); } } - if node_offsets[node_offsets.len() - 1] as usize > selectors_length { + if node_offsets[node_offsets.len() - 1] > selectors_length { return Err(PyValueError::new_err( "node_offsets are out of range of the selectors", )); @@ -283,9 +285,7 @@ where Zip::from(paths.view_mut()) .and(data.rows()) .par_for_each(|path, sample| { - for (tree_start, tree_end) in - node_offsets.iter().map(|i| *i as usize).tuple_windows() - { + for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; @@ -345,7 +345,7 @@ where let leaf_count = *leaf_offsets .last() - .expect("leaf_offsets array cannot be empty") as usize; + .expect("leaf_offsets array cannot be empty"); let mut values = vec![0.0; leaf_count]; let values_iter = MutSlices::new(&mut values, leaf_offsets); @@ -356,7 +356,7 @@ where .install(|| { node_offsets .iter() - .map(|i| *i as usize) + .copied() .tuple_windows() .zip(values_iter) .zip(leaf_offsets) @@ -390,7 +390,7 @@ pub(crate) fn calc_feature_delta_sum<'py>( node_offsets: Bound<'py, PyArray1>, data: Data<'py>, num_threads: usize, -) -> PyResult<(Bound<'py, PyArray2>, Bound<'py, PyArray2>)> { +) -> PyResult> { data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads) } @@ -419,9 +419,7 @@ where .and(delta_sum.rows_mut()) .and(hit_count.rows_mut()) .par_for_each(|sample, mut delta_sum_row, mut hit_count_row| { - for (tree_start, tree_end) in - node_offsets.iter().map(|i| *i as usize).tuple_windows() - { + for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; From 1b3f0d7b66686e38a2f2ee69e63e5800eba95ad5 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 21 May 2024 15:33:11 -0400 Subject: [PATCH 10/40] Better optimized release module --- rust/Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index fa2378f..9762a24 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -11,6 +11,11 @@ crate-type = ["cdylib"] [profile.dev] opt-level = 3 +# Makes linking slower, but the resulting extension module is faster +[profile.release] +lto = true +codegen-units = 1 + [features] default = ["pyo3/abi3-py39"] From b82364a0dd6237c01162f6441c5b78a4292cd751 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 22 May 2024 10:07:17 -0400 Subject: [PATCH 11/40] Allocate output arrays with numpy --- rust/src/tree_traversal.rs | 132 +++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 51 deletions(-) diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index e127ec3..83ac425 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -2,9 +2,9 @@ use crate::mut_slices::MutSlices; use crate::selector::Selector; use enum_dispatch::enum_dispatch; use itertools::Itertools; -use ndarray::{Array1, Array2, ArrayView1, ArrayView2, Axis, Zip}; +use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Axis, Zip}; use num_traits::AsPrimitive; -use numpy::{Element, PyArray, PyArray1, PyArray2, PyArrayMethods}; +use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; @@ -70,15 +70,22 @@ where let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); - // Here we need to dispatch `data` and run the template function - let values = calc_paths_sum_impl( - selectors_view, - node_offsets_view, - data_view, - weights_view, - num_threads, - ); - Ok(PyArray::from_owned_array_bound(py, values)) + Ok({ + let paths = PyArray1::zeros_bound(py, data_view.nrows(), false); + // SAFETY: this call invalidates other views, but it is the only view we need + let paths_view_mut = unsafe { paths.as_array_mut() }; + + // Here we need to dispatch `data` and run the template function + calc_paths_sum_impl( + selectors_view, + node_offsets_view, + data_view, + weights_view, + num_threads, + paths_view_mut, + ); + paths + }) } fn calc_paths_sum_transpose( @@ -109,16 +116,30 @@ where let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); - // Here we need to dispatch `data` and run the template function - let values = calc_paths_sum_transpose_impl( - selectors_view, - node_offsets_view, - leaf_offsets_view, - data_view, - weights_view, - num_threads, - ); - Ok(PyArray::from_owned_array_bound(py, values)) + Ok({ + let values = PyArray1::zeros_bound( + py, + *leaf_offsets_view + .last() + .expect("leaf_offsets array must not be empty"), + false, + ); + + // SAFETY: this call invalidates other views, but it is the only view we need + let values_view = unsafe { values.as_array_mut() }; + + // Here we need to dispatch `data` and run the template function + calc_paths_sum_transpose_impl( + selectors_view, + node_offsets_view, + leaf_offsets_view, + data_view, + weights_view, + num_threads, + values_view, + ); + values + }) } fn calc_feature_delta_sum( @@ -140,13 +161,28 @@ where let data_view = data.as_array(); check_data(data_view)?; - let (delta_sum, hit_count) = - calc_feature_delta_sum_impl(selectors_view, node_offsets_view, data_view, num_threads); - - let delta_sum = PyArray::from_owned_array_bound(py, delta_sum); - let hit_count = PyArray::from_owned_array_bound(py, hit_count); - - Ok((delta_sum, hit_count)) + Ok({ + let delta_sum = + PyArray2::zeros_bound(py, (data_view.nrows(), data_view.ncols()), false); + let hit_count = + PyArray2::zeros_bound(py, (data_view.nrows(), data_view.ncols()), false); + + // SAFETY: this call invalidates other views, but it is the only view we need + let delta_sum_view = unsafe { delta_sum.as_array_mut() }; + // SAFETY: this call invalidates other views, but it is the only view we need + let hit_count_view = unsafe { hit_count.as_array_mut() }; + + calc_feature_delta_sum_impl( + selectors_view, + node_offsets_view, + data_view, + num_threads, + delta_sum_view, + hit_count_view, + ); + + (delta_sum, hit_count) + }) } } @@ -216,6 +252,9 @@ fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) #[inline] fn check_leaf_offsets(leaf_offsets: ArrayView1, node_offset_len: usize) -> PyResult<()> { + if leaf_offsets.len() == 0 { + return Err(PyValueError::new_err("leaf_offsets must not be empty")); + } if leaf_offsets.len() != node_offset_len { return Err(PyValueError::new_err( "leaf_offsets must have the same length as node_offsets", @@ -267,13 +306,11 @@ fn calc_paths_sum_impl( data: ArrayView2, weights: Option>, num_threads: usize, -) -> Array1 -where + paths: ArrayViewMut1, +) where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { - let mut paths = Array1::zeros(data.nrows()); - let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); @@ -282,7 +319,7 @@ where .build() .expect("Cannot build rayon ThreadPool") .install(|| { - Zip::from(paths.view_mut()) + Zip::from(paths) .and(data.rows()) .par_for_each(|path, sample| { for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { @@ -299,8 +336,6 @@ where } }) }); - - paths } #[pyfunction] @@ -331,8 +366,8 @@ fn calc_paths_sum_transpose_impl( data: ArrayView2, weights: Option>, num_threads: usize, -) -> Array1 -where + mut values: ArrayViewMut1, +) where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { @@ -343,11 +378,12 @@ where .as_slice() .expect("Cannot get leaf_offsets slice from ArrayView"); - let leaf_count = *leaf_offsets - .last() - .expect("leaf_offsets array cannot be empty"); - let mut values = vec![0.0; leaf_count]; - let values_iter = MutSlices::new(&mut values, leaf_offsets); + let values_iter = MutSlices::new( + values + .as_slice_mut() + .expect("values must be contiguous and in memory order"), + leaf_offsets, + ); rayon::ThreadPoolBuilder::new() .num_threads(num_threads) @@ -378,8 +414,6 @@ where } }) }); - - values.into() } #[pyfunction] @@ -399,17 +433,15 @@ fn calc_feature_delta_sum_impl( node_offsets: ArrayView1, data: ArrayView2, num_threads: usize, -) -> (Array2, Array2) -where + mut delta_sum: ArrayViewMut2, + mut hit_count: ArrayViewMut2, +) where T: Copy + Send + Sync + PartialOrd + 'static, f64: AsPrimitive, { let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); - let mut delta_sum = Array2::zeros((data.nrows(), data.ncols())); - let mut hit_count = Array2::zeros((data.nrows(), data.ncols())); - rayon::ThreadPoolBuilder::new() .num_threads(num_threads) .build() @@ -452,6 +484,4 @@ where } }); }); - - (delta_sum, hit_count) } From 6d43ff05985841da187d6bae5f5c4cb35e66d224 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 22 May 2024 11:34:06 -0400 Subject: [PATCH 12/40] Change signature impl to match Cython --- rust/src/tree_traversal.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 83ac425..01d2dd6 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -474,10 +474,13 @@ fn calc_feature_delta_sum_impl( }; let child_selector = unsafe { tree_selectors.get_unchecked(i) }; + // Here we cast to f64 following the original Cython implementation, but + // it is a subject to change. *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += 1.0 + 2.0 - * (child_selector.log_n_node_samples as f64 - - parent_selector.log_n_node_samples as f64); + * (child_selector.log_n_node_samples + - parent_selector.log_n_node_samples) + as f64; *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += 1; } From b12e1687add2c1bee755896aa383f056606984b0 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 22 May 2024 15:20:39 -0400 Subject: [PATCH 13/40] Do not create Rayon pool for n_jobs=1 --- rust/src/tree_traversal.rs | 206 ++++++++++++++++++++----------------- 1 file changed, 110 insertions(+), 96 deletions(-) diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 01d2dd6..3861052 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -314,28 +314,33 @@ fn calc_paths_sum_impl( let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - Zip::from(paths) - .and(data.rows()) - .par_for_each(|path, sample| { - for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { - let tree_selectors = - unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - if let Some(weights) = weights { - *path += *unsafe { weights.uget(leaf.left as usize) } * leaf.value; - } else { - *path += leaf.value; - } - } - }) - }); + let inner_fn = |path: &mut f64, sample: ArrayView1| { + for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { + let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + if let Some(weights) = weights { + *path += *unsafe { weights.uget(leaf.left as usize) } * leaf.value; + } else { + *path += leaf.value; + } + } + }; + + let zip = Zip::from(paths).and(data.rows()); + + if num_threads == 1 { + zip.for_each(inner_fn); + } else { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + zip.par_for_each(inner_fn); + }); + } } #[pyfunction] @@ -385,35 +390,40 @@ fn calc_paths_sum_transpose_impl( leaf_offsets, ); - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - node_offsets - .iter() - .copied() - .tuple_windows() - .zip(values_iter) - .zip(leaf_offsets) - .par_bridge() - .for_each(|(((tree_start, tree_end), values), &leaf_offset)| { - for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { - let tree_selectors = - unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - let value = - unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; - if let Some(weights) = weights { - *value += weights[x_index] * leaf.value; - } else { - *value += leaf.value; - } - } - }) - }); + let inner_fn = + |(((tree_start, tree_end), values), &leaf_offset): (((usize, usize), &mut [f64]), _)| { + for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { + let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + let value = unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; + if let Some(weights) = weights { + *value += weights[x_index] * leaf.value; + } else { + *value += leaf.value; + } + } + }; + + let iter = node_offsets + .iter() + .copied() + .tuple_windows() + .zip(values_iter) + .zip(leaf_offsets); + + if num_threads == 1 { + iter.for_each(inner_fn); + } else { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + iter.par_bridge().for_each(inner_fn); + }); + } } #[pyfunction] @@ -442,49 +452,53 @@ fn calc_feature_delta_sum_impl( let node_offsets = node_offsets.as_slice().unwrap(); let selectors = selectors.as_slice().unwrap(); - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - Zip::from(data.rows()) - .and(delta_sum.rows_mut()) - .and(hit_count.rows_mut()) - .par_for_each(|sample, mut delta_sum_row, mut hit_count_row| { - for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { - let tree_selectors = - unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let mut i = 0; - let mut parent_selector: &Selector; - loop { - parent_selector = unsafe { tree_selectors.get_unchecked(i) }; - if parent_selector.is_leaf() { - break; - } - - // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? - let threshold: T = parent_selector.value.as_(); - i = if *unsafe { sample.uget(parent_selector.feature as usize) } - <= threshold - { - parent_selector.left as usize - } else { - parent_selector.right as usize - }; - - let child_selector = unsafe { tree_selectors.get_unchecked(i) }; - // Here we cast to f64 following the original Cython implementation, but - // it is a subject to change. - *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += - 1.0 + 2.0 - * (child_selector.log_n_node_samples - - parent_selector.log_n_node_samples) - as f64; - *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += - 1; - } - } - }); - }); + let inner_fn = |sample: ArrayView1, + mut delta_sum_row: ArrayViewMut1, + mut hit_count_row: ArrayViewMut1| { + for (tree_start, tree_end) in node_offsets.iter().copied().tuple_windows() { + let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let mut i = 0; + let mut parent_selector: &Selector; + loop { + parent_selector = unsafe { tree_selectors.get_unchecked(i) }; + if parent_selector.is_leaf() { + break; + } + + // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? + let threshold: T = parent_selector.value.as_(); + i = if *unsafe { sample.uget(parent_selector.feature as usize) } <= threshold { + parent_selector.left as usize + } else { + parent_selector.right as usize + }; + + let child_selector = unsafe { tree_selectors.get_unchecked(i) }; + // Here we cast to f64 following the original Cython implementation, but + // it is a subject to change. + *unsafe { delta_sum_row.uget_mut(parent_selector.feature as usize) } += 1.0 + + 2.0 + * (child_selector.log_n_node_samples - parent_selector.log_n_node_samples) + as f64; + *unsafe { hit_count_row.uget_mut(parent_selector.feature as usize) } += 1; + } + } + }; + + let zip = Zip::from(data.rows()) + .and(delta_sum.rows_mut()) + .and(hit_count.rows_mut()); + + if num_threads == 1 { + zip.for_each(inner_fn); + } else { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + zip.par_for_each(inner_fn); + }); + } } From 995d29d611369d9bdaf6b9993390b4a36aa8e712 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Sat, 25 May 2024 07:52:53 -0400 Subject: [PATCH 14/40] Simgle/multi impls for transpose --- .../calc_paths_sum_transpose_impl.rs | 125 ++++++++++++++++++ rust/src/tree_traversal/find_leaf.rs | 26 ++++ .../mod.rs} | 95 +------------ tests/test_aadforest.py | 12 +- 4 files changed, 163 insertions(+), 95 deletions(-) create mode 100644 rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs create mode 100644 rust/src/tree_traversal/find_leaf.rs rename rust/src/{tree_traversal.rs => tree_traversal/mod.rs} (83%) diff --git a/rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs b/rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs new file mode 100644 index 0000000..6f1637a --- /dev/null +++ b/rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs @@ -0,0 +1,125 @@ +use crate::mut_slices::MutSlices; +use crate::selector::Selector; +use crate::tree_traversal::find_leaf::find_leaf; +use itertools::{Either, Itertools}; +use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, Axis}; +use num_traits::AsPrimitive; +use rayon::prelude::*; +use std::iter::repeat; + +pub(super) fn calc_paths_sum_transpose_impl( + selectors: ArrayView1, + node_offsets: ArrayView1, + leaf_offsets: ArrayView1, + data: ArrayView2, + weights: Option>, + num_threads: usize, + mut values: ArrayViewMut1, +) where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let selectors = selectors + .as_slice() + .expect("selectors must be contiguous and in memory order"); + let leaf_offsets = leaf_offsets + .as_slice() + .expect("leaf_offsets must be contiguous and in memory order"); + let weights = weights.as_ref().map(|array_view| { + array_view + .as_slice() + .expect("weights must be contiguous and in memory order") + }); + let values = values + .as_slice_mut() + .expect("values must be contiguous and in memory order"); + + if num_threads == 1 { + single_thread(selectors, node_offsets, data, weights, values) + } else { + multithread( + selectors, + node_offsets, + leaf_offsets, + data, + weights, + num_threads, + values, + ) + } +} + +fn single_thread( + selectors: &[Selector], + node_offsets: ArrayView1, + data: ArrayView2, + weights: Option<&[f64]>, + values: &mut [f64], +) where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + for (sample, weight) in data.axis_iter(Axis(0)).zip(weights_iterator(weights)) { + for tree_range in node_offsets.iter().copied().tuple_windows() { + update_values(selectors, sample, weight, tree_range, values, 0) + } + } +} + +fn multithread( + selectors: &[Selector], + node_offsets: ArrayView1, + leaf_offsets: &[usize], + data: ArrayView2, + weights: Option<&[f64]>, + num_threads: usize, + values: &mut [f64], +) where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let values_iter = MutSlices::new(values, leaf_offsets); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + node_offsets + .iter() + .copied() + .tuple_windows() + .zip(values_iter) + .zip(leaf_offsets) + .par_bridge() + .for_each(|((tree_range, values), &leaf_offset)| { + for (sample, weight) in data.axis_iter(Axis(0)).zip(weights_iterator(weights)) { + update_values(selectors, sample, weight, tree_range, values, leaf_offset) + } + }); + }); +} + +fn weights_iterator(weights: Option<&[f64]>) -> impl Iterator + '_ { + match weights { + Some(weights) => Either::Left(weights.iter().copied()), + None => Either::Right(repeat(1.0)), + } +} + +fn update_values( + selectors: &[Selector], + sample: ArrayView1, + weight: f64, + tree_range: (usize, usize), + values: &mut [f64], + leaf_offset: usize, +) where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let (tree_start, tree_end) = tree_range; + let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + *unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) } += weight * leaf.value; +} diff --git a/rust/src/tree_traversal/find_leaf.rs b/rust/src/tree_traversal/find_leaf.rs new file mode 100644 index 0000000..9fe3e1e --- /dev/null +++ b/rust/src/tree_traversal/find_leaf.rs @@ -0,0 +1,26 @@ +use crate::selector::Selector; +use num_traits::AsPrimitive; + +// It looks like the performance is not affected by returning a copy of Selector, not reference. +#[inline] +pub(super) fn find_leaf(tree: &[Selector], sample: &[T]) -> Selector +where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let mut i = 0; + loop { + let selector = *unsafe { tree.get_unchecked(i) }; + if selector.is_leaf() { + break selector; + } + + // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? + let threshold: T = selector.value.as_(); + i = if *unsafe { sample.get_unchecked(selector.feature as usize) } <= threshold { + selector.left as usize + } else { + selector.right as usize + }; + } +} diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal/mod.rs similarity index 83% rename from rust/src/tree_traversal.rs rename to rust/src/tree_traversal/mod.rs index 3861052..2de7f9e 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal/mod.rs @@ -1,14 +1,17 @@ -use crate::mut_slices::MutSlices; use crate::selector::Selector; +use crate::tree_traversal::calc_paths_sum_transpose_impl::calc_paths_sum_transpose_impl; +use crate::tree_traversal::find_leaf::find_leaf; use enum_dispatch::enum_dispatch; use itertools::Itertools; -use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Axis, Zip}; +use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Zip}; use num_traits::AsPrimitive; use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; -use rayon::iter::{ParallelBridge, ParallelIterator}; + +mod calc_paths_sum_transpose_impl; +mod find_leaf; type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); @@ -193,30 +196,6 @@ pub(crate) enum Data<'py> { F32(Bound<'py, PyArray2>), } -// It looks like the performance is not affected by returning a copy of Selector, not reference. -#[inline] -fn find_leaf(tree: &[Selector], sample: &[T]) -> Selector -where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let mut i = 0; - loop { - let selector = *unsafe { tree.get_unchecked(i) }; - if selector.is_leaf() { - break selector; - } - - // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? - let threshold: T = selector.value.as_(); - i = if *unsafe { sample.get_unchecked(selector.feature as usize) } <= threshold { - selector.left as usize - } else { - selector.right as usize - }; - } -} - #[inline] fn check_selectors(selectors: ArrayView1) -> PyResult<()> { if !selectors.is_standard_layout() { @@ -364,68 +343,6 @@ pub(crate) fn calc_paths_sum_transpose<'py>( ) } -fn calc_paths_sum_transpose_impl( - selectors: ArrayView1, - node_offsets: ArrayView1, - leaf_offsets: ArrayView1, - data: ArrayView2, - weights: Option>, - num_threads: usize, - mut values: ArrayViewMut1, -) where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let selectors = selectors - .as_slice() - .expect("Cannot get selectors slice from ArrayView"); - let leaf_offsets = leaf_offsets - .as_slice() - .expect("Cannot get leaf_offsets slice from ArrayView"); - - let values_iter = MutSlices::new( - values - .as_slice_mut() - .expect("values must be contiguous and in memory order"), - leaf_offsets, - ); - - let inner_fn = - |(((tree_start, tree_end), values), &leaf_offset): (((usize, usize), &mut [f64]), _)| { - for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { - let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; - - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - - let value = unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; - if let Some(weights) = weights { - *value += weights[x_index] * leaf.value; - } else { - *value += leaf.value; - } - } - }; - - let iter = node_offsets - .iter() - .copied() - .tuple_windows() - .zip(values_iter) - .zip(leaf_offsets); - - if num_threads == 1 { - iter.for_each(inner_fn); - } else { - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - iter.par_bridge().for_each(inner_fn); - }); - } -} - #[pyfunction] #[pyo3(signature = (selectors, node_offsets, data, num_threads = 0))] pub(crate) fn calc_feature_delta_sum<'py>( diff --git a/tests/test_aadforest.py b/tests/test_aadforest.py index 331f364..63a9661 100644 --- a/tests/test_aadforest.py +++ b/tests/test_aadforest.py @@ -41,11 +41,12 @@ def test_prior_influence_callable(): assert np.argmin(scores) == data.shape[0] - 1 -# Single-thread and parallel implementations are a bit different, so here we check both. -# We use n_thread parameter instead of n_jobs, which is a fixture in conftest.py -@pytest.mark.parametrize("n_thread", [1, 2]) +# Rust implementations for single and multithread differ, we must test both +# We call function parameter n_threads to be not confused with n_jobs we use +# as a fixture. +@pytest.mark.parametrize("n_threads", [1, 2]) @pytest.mark.regression -def test_regression_fit_known(n_thread, regression_data): +def test_regression_fit_known(n_threads, regression_data): random_seed = 0 n_samples = 1024 n_features = 16 @@ -56,8 +57,7 @@ def test_regression_fit_known(n_thread, regression_data): known_data = data[rng.choice(n_samples, n_known, replace=False)] known_labels = rng.choice([-1, 1], n_known, replace=True) - # This small sampletrees_per_batch is inefficient, but it's good for testing to guarantee parallel execution. - forest = AADForest(n_trees=n_trees, random_seed=random_seed, n_jobs=n_thread, sampletrees_per_batch=2048) + forest = AADForest(n_trees=n_trees, random_seed=random_seed, n_jobs=n_threads) forest.fit(data) pre_fit_known_scores = forest.score_samples(data) From 13cb048389ad6eca28d954454deb625dd0e5c5ed Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Sat, 25 May 2024 16:35:18 -0400 Subject: [PATCH 15/40] Revert "Simgle/multi impls for transpose" This reverts commit f85056db16dc578926a75c937abf593217919260. --- .../mod.rs => tree_traversal.rs} | 95 ++++++++++++- .../calc_paths_sum_transpose_impl.rs | 125 ------------------ rust/src/tree_traversal/find_leaf.rs | 26 ---- tests/test_aadforest.py | 8 +- 4 files changed, 91 insertions(+), 163 deletions(-) rename rust/src/{tree_traversal/mod.rs => tree_traversal.rs} (83%) delete mode 100644 rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs delete mode 100644 rust/src/tree_traversal/find_leaf.rs diff --git a/rust/src/tree_traversal/mod.rs b/rust/src/tree_traversal.rs similarity index 83% rename from rust/src/tree_traversal/mod.rs rename to rust/src/tree_traversal.rs index 2de7f9e..3861052 100644 --- a/rust/src/tree_traversal/mod.rs +++ b/rust/src/tree_traversal.rs @@ -1,17 +1,14 @@ +use crate::mut_slices::MutSlices; use crate::selector::Selector; -use crate::tree_traversal::calc_paths_sum_transpose_impl::calc_paths_sum_transpose_impl; -use crate::tree_traversal::find_leaf::find_leaf; use enum_dispatch::enum_dispatch; use itertools::Itertools; -use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Zip}; +use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Axis, Zip}; use num_traits::AsPrimitive; use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; - -mod calc_paths_sum_transpose_impl; -mod find_leaf; +use rayon::iter::{ParallelBridge, ParallelIterator}; type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); @@ -196,6 +193,30 @@ pub(crate) enum Data<'py> { F32(Bound<'py, PyArray2>), } +// It looks like the performance is not affected by returning a copy of Selector, not reference. +#[inline] +fn find_leaf(tree: &[Selector], sample: &[T]) -> Selector +where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let mut i = 0; + loop { + let selector = *unsafe { tree.get_unchecked(i) }; + if selector.is_leaf() { + break selector; + } + + // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? + let threshold: T = selector.value.as_(); + i = if *unsafe { sample.get_unchecked(selector.feature as usize) } <= threshold { + selector.left as usize + } else { + selector.right as usize + }; + } +} + #[inline] fn check_selectors(selectors: ArrayView1) -> PyResult<()> { if !selectors.is_standard_layout() { @@ -343,6 +364,68 @@ pub(crate) fn calc_paths_sum_transpose<'py>( ) } +fn calc_paths_sum_transpose_impl( + selectors: ArrayView1, + node_offsets: ArrayView1, + leaf_offsets: ArrayView1, + data: ArrayView2, + weights: Option>, + num_threads: usize, + mut values: ArrayViewMut1, +) where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let selectors = selectors + .as_slice() + .expect("Cannot get selectors slice from ArrayView"); + let leaf_offsets = leaf_offsets + .as_slice() + .expect("Cannot get leaf_offsets slice from ArrayView"); + + let values_iter = MutSlices::new( + values + .as_slice_mut() + .expect("values must be contiguous and in memory order"), + leaf_offsets, + ); + + let inner_fn = + |(((tree_start, tree_end), values), &leaf_offset): (((usize, usize), &mut [f64]), _)| { + for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { + let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; + + let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); + + let value = unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; + if let Some(weights) = weights { + *value += weights[x_index] * leaf.value; + } else { + *value += leaf.value; + } + } + }; + + let iter = node_offsets + .iter() + .copied() + .tuple_windows() + .zip(values_iter) + .zip(leaf_offsets); + + if num_threads == 1 { + iter.for_each(inner_fn); + } else { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + iter.par_bridge().for_each(inner_fn); + }); + } +} + #[pyfunction] #[pyo3(signature = (selectors, node_offsets, data, num_threads = 0))] pub(crate) fn calc_feature_delta_sum<'py>( diff --git a/rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs b/rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs deleted file mode 100644 index 6f1637a..0000000 --- a/rust/src/tree_traversal/calc_paths_sum_transpose_impl.rs +++ /dev/null @@ -1,125 +0,0 @@ -use crate::mut_slices::MutSlices; -use crate::selector::Selector; -use crate::tree_traversal::find_leaf::find_leaf; -use itertools::{Either, Itertools}; -use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, Axis}; -use num_traits::AsPrimitive; -use rayon::prelude::*; -use std::iter::repeat; - -pub(super) fn calc_paths_sum_transpose_impl( - selectors: ArrayView1, - node_offsets: ArrayView1, - leaf_offsets: ArrayView1, - data: ArrayView2, - weights: Option>, - num_threads: usize, - mut values: ArrayViewMut1, -) where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let selectors = selectors - .as_slice() - .expect("selectors must be contiguous and in memory order"); - let leaf_offsets = leaf_offsets - .as_slice() - .expect("leaf_offsets must be contiguous and in memory order"); - let weights = weights.as_ref().map(|array_view| { - array_view - .as_slice() - .expect("weights must be contiguous and in memory order") - }); - let values = values - .as_slice_mut() - .expect("values must be contiguous and in memory order"); - - if num_threads == 1 { - single_thread(selectors, node_offsets, data, weights, values) - } else { - multithread( - selectors, - node_offsets, - leaf_offsets, - data, - weights, - num_threads, - values, - ) - } -} - -fn single_thread( - selectors: &[Selector], - node_offsets: ArrayView1, - data: ArrayView2, - weights: Option<&[f64]>, - values: &mut [f64], -) where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - for (sample, weight) in data.axis_iter(Axis(0)).zip(weights_iterator(weights)) { - for tree_range in node_offsets.iter().copied().tuple_windows() { - update_values(selectors, sample, weight, tree_range, values, 0) - } - } -} - -fn multithread( - selectors: &[Selector], - node_offsets: ArrayView1, - leaf_offsets: &[usize], - data: ArrayView2, - weights: Option<&[f64]>, - num_threads: usize, - values: &mut [f64], -) where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let values_iter = MutSlices::new(values, leaf_offsets); - - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build() - .expect("Cannot build rayon ThreadPool") - .install(|| { - node_offsets - .iter() - .copied() - .tuple_windows() - .zip(values_iter) - .zip(leaf_offsets) - .par_bridge() - .for_each(|((tree_range, values), &leaf_offset)| { - for (sample, weight) in data.axis_iter(Axis(0)).zip(weights_iterator(weights)) { - update_values(selectors, sample, weight, tree_range, values, leaf_offset) - } - }); - }); -} - -fn weights_iterator(weights: Option<&[f64]>) -> impl Iterator + '_ { - match weights { - Some(weights) => Either::Left(weights.iter().copied()), - None => Either::Right(repeat(1.0)), - } -} - -fn update_values( - selectors: &[Selector], - sample: ArrayView1, - weight: f64, - tree_range: (usize, usize), - values: &mut [f64], - leaf_offset: usize, -) where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let (tree_start, tree_end) = tree_range; - let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; - let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - *unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) } += weight * leaf.value; -} diff --git a/rust/src/tree_traversal/find_leaf.rs b/rust/src/tree_traversal/find_leaf.rs deleted file mode 100644 index 9fe3e1e..0000000 --- a/rust/src/tree_traversal/find_leaf.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::selector::Selector; -use num_traits::AsPrimitive; - -// It looks like the performance is not affected by returning a copy of Selector, not reference. -#[inline] -pub(super) fn find_leaf(tree: &[Selector], sample: &[T]) -> Selector -where - T: Copy + Send + Sync + PartialOrd + 'static, - f64: AsPrimitive, -{ - let mut i = 0; - loop { - let selector = *unsafe { tree.get_unchecked(i) }; - if selector.is_leaf() { - break selector; - } - - // TODO: do opposite type casting: what if we trained on huge f64 and predict on f32? - let threshold: T = selector.value.as_(); - i = if *unsafe { sample.get_unchecked(selector.feature as usize) } <= threshold { - selector.left as usize - } else { - selector.right as usize - }; - } -} diff --git a/tests/test_aadforest.py b/tests/test_aadforest.py index 63a9661..f2a254d 100644 --- a/tests/test_aadforest.py +++ b/tests/test_aadforest.py @@ -41,12 +41,8 @@ def test_prior_influence_callable(): assert np.argmin(scores) == data.shape[0] - 1 -# Rust implementations for single and multithread differ, we must test both -# We call function parameter n_threads to be not confused with n_jobs we use -# as a fixture. -@pytest.mark.parametrize("n_threads", [1, 2]) @pytest.mark.regression -def test_regression_fit_known(n_threads, regression_data): +def test_regression_fit_known(regression_data): random_seed = 0 n_samples = 1024 n_features = 16 @@ -57,7 +53,7 @@ def test_regression_fit_known(n_threads, regression_data): known_data = data[rng.choice(n_samples, n_known, replace=False)] known_labels = rng.choice([-1, 1], n_known, replace=True) - forest = AADForest(n_trees=n_trees, random_seed=random_seed, n_jobs=n_threads) + forest = AADForest(n_trees=n_trees, random_seed=random_seed) forest.fit(data) pre_fit_known_scores = forest.score_samples(data) From 73869347a34a501d96a23d574ea8be2250decfb0 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Mon, 27 May 2024 08:39:14 -0400 Subject: [PATCH 16/40] Parallel batch_size --- rust/src/mut_slices.rs | 216 +++++++++++++++++++++++++++++++++--- rust/src/tree_traversal.rs | 109 ++++++++++++++---- src/coniferest/aadforest.py | 16 ++- src/coniferest/evaluator.py | 31 ++++-- tests/test_aadforest.py | 8 +- 5 files changed, 331 insertions(+), 49 deletions(-) diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index 22ae09d..c66814f 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -1,16 +1,14 @@ +use rayon::iter::plumbing::{bridge, Consumer, Producer, ProducerCallback, UnindexedConsumer}; +use rayon::iter::{IndexedParallelIterator, ParallelIterator}; + pub struct MutSlices<'sl, 'off, T> { slice: &'sl mut [T], offsets: &'off [usize], - current: usize, } impl<'sl, 'off, T> MutSlices<'sl, 'off, T> { pub fn new(slice: &'sl mut [T], offsets: &'off [usize]) -> Self { - MutSlices { - slice, - offsets, - current: 0, - } + MutSlices { slice, offsets } } } @@ -18,17 +16,209 @@ impl<'sl, 'off, T> Iterator for MutSlices<'sl, 'off, T> { type Item = &'sl mut [T]; fn next(&mut self) -> Option { - if self.current >= self.offsets.len() - 1 { + if self.offsets.len() <= 1 { return None; } - let start = self.offsets[self.current]; - let end = self.offsets[self.current + 1]; - self.current += 1; - - // Here we temporarily replace slice with an empty one - let (left, right) = std::mem::take(&mut self.slice).split_at_mut(end - start); + // Split slice, save right. Here we temporarily replace slice with an empty one + let (left, right) = + std::mem::take(&mut self.slice).split_at_mut(self.offsets[1] - self.offsets[0]); self.slice = right; + + // Move offsets to the right + self.offsets = &self.offsets[1..]; + Some(left) } + + fn size_hint(&self) -> (usize, Option) { + let len = self.offsets.len() - 1; + (len, Some(len)) + } +} + +impl<'sl, 'off, T> DoubleEndedIterator for MutSlices<'sl, 'off, T> { + fn next_back(&mut self) -> Option { + let offsets_len = self.offsets.len(); + if offsets_len <= 1 { + return None; + } + + // Split slice, save left. Here we temporarily replace slice with an empty one + let (left, right) = std::mem::take(&mut self.slice) + .split_at_mut(self.offsets[offsets_len - 2] - self.offsets[0]); + self.slice = left; + + // Move offsets to the left + self.offsets = &self.offsets[..offsets_len - 1]; + + Some(right) + } +} + +impl<'sl, 'off, T> ExactSizeIterator for MutSlices<'sl, 'off, T> { + fn len(&self) -> usize { + self.offsets.len() - 1 + } +} + +// Following rayon's ChunksMut implementation +impl<'sl, 'off, T> ParallelIterator for MutSlices<'sl, 'off, T> +where + T: Send, +{ + type Item = &'sl mut [T]; + + fn drive_unindexed(self, consumer: C) -> C::Result + where + C: UnindexedConsumer, + { + bridge(self, consumer) + } + + fn opt_len(&self) -> Option { + ExactSizeIterator::len(self).into() + } +} + +impl<'sl, 'off, T> IndexedParallelIterator for MutSlices<'sl, 'off, T> +where + T: Send, +{ + fn len(&self) -> usize { + ExactSizeIterator::len(self) + } + + fn drive(self, consumer: C) -> C::Result + where + C: Consumer, + { + bridge(self, consumer) + } + + fn with_producer(self, callback: CB) -> CB::Output + where + CB: ProducerCallback, + { + callback.callback(MutSlicesProducer { + slice: self.slice, + offsets: self.offsets, + }) + } +} + +struct MutSlicesProducer<'sl, 'off, T> { + slice: &'sl mut [T], + offsets: &'off [usize], +} + +impl<'sl, 'off, T> Producer for MutSlicesProducer<'sl, 'off, T> +where + T: Send, +{ + type Item = &'sl mut [T]; + type IntoIter = MutSlices<'sl, 'off, T>; + + fn into_iter(self) -> Self::IntoIter { + MutSlices::new(self.slice, self.offsets) + } + + fn split_at(self, index: usize) -> (Self, Self) { + let (left_slice, right_slice) = self + .slice + .split_at_mut(self.offsets[index] - self.offsets[0]); + let (left_offsets, right_offsets) = (&self.offsets[..=index], &self.offsets[index..]); + ( + MutSlicesProducer { + slice: left_slice, + offsets: left_offsets, + }, + MutSlicesProducer { + slice: right_slice, + offsets: right_offsets, + }, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use itertools::Itertools; + + #[test] + fn test_mut_slices() { + let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let offsets = vec![0, 3, 6, 10]; + + let mut slices = MutSlices::new(&mut data, &offsets); + + assert_eq!(slices.next().unwrap(), &[1, 2, 3]); + assert_eq!(slices.next().unwrap(), &[4, 5, 6]); + assert_eq!(slices.next().unwrap(), &[7, 8, 9, 10]); + assert!(slices.next().is_none()); + + let mut slices = MutSlices::new(&mut data, &offsets); + + assert_eq!(slices.next_back().unwrap(), &[7, 8, 9, 10]); + assert_eq!(slices.next_back().unwrap(), &[4, 5, 6]); + assert_eq!(slices.next_back().unwrap(), &[1, 2, 3]); + assert!(slices.next_back().is_none()); + + let mut slices = MutSlices::new(&mut data, &offsets); + assert_eq!(slices.next().unwrap(), &[1, 2, 3]); + assert_eq!(slices.next_back().unwrap(), &[7, 8, 9, 10]); + assert_eq!(slices.next().unwrap(), &[4, 5, 6]); + assert_eq!(slices.next_back(), None); + assert_eq!(slices.next(), None); + } + + #[test] + fn test_mut_slices_len() { + let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let offsets = vec![0, 3, 6, 10]; + + let slices = MutSlices::new(&mut data, &offsets); + + assert_eq!(ExactSizeIterator::len(&slices), 3); + } + + #[test] + fn test_mut_slices_parallel() { + let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let offsets = vec![0, 3, 6, 10]; + + let slices = MutSlices::new(&mut data, &offsets); + + let sum: usize = ParallelIterator::map(slices, |slice| slice.iter().sum::()).sum(); + + assert_eq!(sum, data.iter().sum()); + } + + #[test] + fn test_mut_slices_producer() { + let mut data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let offsets = vec![0, 3, 6, 10]; + + let producer = MutSlicesProducer { + slice: &mut data, + offsets: &offsets, + }; + assert_eq!( + producer.into_iter().collect_vec(), + [vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9, 10]] + ); + + let producer = MutSlicesProducer { + slice: &mut data, + offsets: &offsets, + }; + let (left, right) = producer.split_at(1); + assert_eq!(left.into_iter().collect_vec(), [&[1, 2, 3]]); + assert_eq!( + right.into_iter().collect_vec(), + [&vec![4, 5, 6], &vec![7, 8, 9, 10]] + ); + } } diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 3861052..27e23e9 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -2,13 +2,14 @@ use crate::mut_slices::MutSlices; use crate::selector::Selector; use enum_dispatch::enum_dispatch; use itertools::Itertools; +use ndarray::parallel::prelude::*; use ndarray::{ArrayView1, ArrayView2, ArrayViewMut1, ArrayViewMut2, Axis, Zip}; use num_traits::AsPrimitive; use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; -use rayon::iter::{ParallelBridge, ParallelIterator}; +use rayon::prelude::*; type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); @@ -21,6 +22,7 @@ trait DataTrait<'py> { node_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, + batch_size: usize, ) -> PyResult>>; fn calc_paths_sum_transpose( @@ -31,6 +33,7 @@ trait DataTrait<'py> { leaf_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, + batch_size: usize, ) -> PyResult>>; fn calc_feature_delta_sum( @@ -39,6 +42,7 @@ trait DataTrait<'py> { selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, num_threads: usize, + batch_size: usize, ) -> PyResult>; } @@ -54,6 +58,7 @@ where node_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, + batch_size: usize, ) -> PyResult>> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); @@ -70,6 +75,8 @@ where let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); + let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; + Ok({ let paths = PyArray1::zeros_bound(py, data_view.nrows(), false); // SAFETY: this call invalidates other views, but it is the only view we need @@ -82,6 +89,7 @@ where data_view, weights_view, num_threads, + batch_size, paths_view_mut, ); paths @@ -96,6 +104,7 @@ where leaf_offsets: Bound<'py, PyArray1>, weights: Option>>, num_threads: usize, + batch_size: usize, ) -> PyResult>> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); @@ -116,6 +125,8 @@ where let weights = weights.map(|weights| weights.readonly()); let weights_view = weights.as_ref().map(|weights| weights.as_array()); + let num_threads = get_num_threads(data_view.ncols(), num_threads, batch_size)?; + Ok({ let values = PyArray1::zeros_bound( py, @@ -136,6 +147,7 @@ where data_view, weights_view, num_threads, + batch_size, values_view, ); values @@ -148,6 +160,7 @@ where selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, num_threads: usize, + batch_size: usize, ) -> PyResult> { let selectors = selectors.readonly(); let selectors_view = selectors.as_array(); @@ -161,6 +174,8 @@ where let data_view = data.as_array(); check_data(data_view)?; + let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; + Ok({ let delta_sum = PyArray2::zeros_bound(py, (data_view.nrows(), data_view.ncols()), false); @@ -177,6 +192,7 @@ where node_offsets_view, data_view, num_threads, + batch_size, delta_sum_view, hit_count_view, ); @@ -229,6 +245,11 @@ fn check_selectors(selectors: ArrayView1) -> PyResult<()> { #[inline] fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) -> PyResult<()> { + if node_offsets.len() <= 1 { + return Err(PyValueError::new_err( + "node_offsets must have at least two elements", + )); + } if let Some(node_offsets) = node_offsets.as_slice() { for (x, y) in node_offsets.iter().copied().tuple_windows() { if x > y { @@ -252,8 +273,10 @@ fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) #[inline] fn check_leaf_offsets(leaf_offsets: ArrayView1, node_offset_len: usize) -> PyResult<()> { - if leaf_offsets.len() == 0 { - return Err(PyValueError::new_err("leaf_offsets must not be empty")); + if leaf_offsets.len() <= 1 { + return Err(PyValueError::new_err( + "leaf_offsets must have at least two elements", + )); } if leaf_offsets.len() != node_offset_len { return Err(PyValueError::new_err( @@ -286,8 +309,18 @@ fn check_data(data: ArrayView2) -> PyResult<()> { Ok(()) } +#[inline] +fn get_num_threads(nrows: usize, num_threads: usize, batch_size: usize) -> PyResult { + if batch_size == 0 { + Err(PyValueError::new_err("batch_size must be greater than 0")) + } else { + let n_jobs = nrows.div_ceil(batch_size); + Ok(usize::min(num_threads, n_jobs)) + } +} + #[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, weights = None, num_threads = 0))] +#[pyo3(signature = (selectors, node_offsets, data, weights = None, *, num_threads, batch_size))] pub(crate) fn calc_paths_sum<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, @@ -296,8 +329,16 @@ pub(crate) fn calc_paths_sum<'py>( data: Data<'py>, weights: Option>>, num_threads: usize, + batch_size: usize, ) -> PyResult>> { - data.calc_paths_sum(py, selectors, node_offsets, weights, num_threads) + data.calc_paths_sum( + py, + selectors, + node_offsets, + weights, + num_threads, + batch_size, + ) } fn calc_paths_sum_impl( @@ -306,6 +347,7 @@ fn calc_paths_sum_impl( data: ArrayView2, weights: Option>, num_threads: usize, + batch_size: usize, paths: ArrayViewMut1, ) where T: Copy + Send + Sync + PartialOrd + 'static, @@ -338,13 +380,15 @@ fn calc_paths_sum_impl( .build() .expect("Cannot build rayon ThreadPool") .install(|| { - zip.par_for_each(inner_fn); + zip.into_par_iter() + .with_min_len(batch_size) + .for_each(|(path, sample)| inner_fn(path, sample)); }); } } #[pyfunction] -#[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, num_threads = 0))] +#[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, *, num_threads, batch_size))] pub(crate) fn calc_paths_sum_transpose<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, @@ -353,6 +397,7 @@ pub(crate) fn calc_paths_sum_transpose<'py>( data: Data<'py>, weights: Option>>, num_threads: usize, + batch_size: usize, ) -> PyResult>> { data.calc_paths_sum_transpose( py, @@ -361,6 +406,7 @@ pub(crate) fn calc_paths_sum_transpose<'py>( leaf_offsets, weights, num_threads, + batch_size, ) } @@ -371,6 +417,7 @@ fn calc_paths_sum_transpose_impl( data: ArrayView2, weights: Option>, num_threads: usize, + batch_size: usize, mut values: ArrayViewMut1, ) where T: Copy + Send + Sync + PartialOrd + 'static, @@ -378,10 +425,13 @@ fn calc_paths_sum_transpose_impl( { let selectors = selectors .as_slice() - .expect("Cannot get selectors slice from ArrayView"); + .expect("selectors must be contiguous and in memory order"); + let node_offsets = node_offsets + .as_slice() + .expect("node_offsets must be contiguous and in memory order"); let leaf_offsets = leaf_offsets .as_slice() - .expect("Cannot get leaf_offsets slice from ArrayView"); + .expect("leaf_offsets must be contiguous and in memory order"); let values_iter = MutSlices::new( values @@ -391,13 +441,13 @@ fn calc_paths_sum_transpose_impl( ); let inner_fn = - |(((tree_start, tree_end), values), &leaf_offset): (((usize, usize), &mut [f64]), _)| { + |(((tree_start, tree_end), values), &leaf_first): (((usize, usize), &mut [f64]), _)| { for (x_index, sample) in data.axis_iter(Axis(0)).enumerate() { let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; let leaf = find_leaf(tree_selectors, sample.as_slice().unwrap()); - let value = unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_offset) }; + let value = unsafe { values.get_unchecked_mut(leaf.left as usize - leaf_first) }; if let Some(weights) = weights { *value += weights[x_index] * leaf.value; } else { @@ -406,36 +456,46 @@ fn calc_paths_sum_transpose_impl( } }; - let iter = node_offsets - .iter() - .copied() - .tuple_windows() - .zip(values_iter) - .zip(leaf_offsets); + let leaf_firsts = &leaf_offsets[..leaf_offsets.len() - 1]; if num_threads == 1 { - iter.for_each(inner_fn); + // Here we use itertools methods + node_offsets + .iter() + .copied() + .tuple_windows() + .zip_eq(values_iter) + .zip_eq(leaf_firsts) + .for_each(inner_fn); } else { rayon::ThreadPoolBuilder::new() .num_threads(num_threads) .build() .expect("Cannot build rayon ThreadPool") .install(|| { - iter.par_bridge().for_each(inner_fn); + // Here we use rayon methods + node_offsets + .par_windows(2) + .map(|window| (window[0], window[1])) + .zip_eq(values_iter) + .zip_eq(leaf_firsts) + .with_min_len(batch_size) + .for_each(inner_fn); }); } } #[pyfunction] -#[pyo3(signature = (selectors, node_offsets, data, num_threads = 0))] +#[pyo3(signature = (selectors, node_offsets, data, *, num_threads, batch_size))] pub(crate) fn calc_feature_delta_sum<'py>( py: Python<'py>, selectors: Bound<'py, PyArray1>, node_offsets: Bound<'py, PyArray1>, data: Data<'py>, num_threads: usize, + batch_size: usize, ) -> PyResult> { - data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads) + data.calc_feature_delta_sum(py, selectors, node_offsets, num_threads, batch_size) } fn calc_feature_delta_sum_impl( @@ -443,6 +503,7 @@ fn calc_feature_delta_sum_impl( node_offsets: ArrayView1, data: ArrayView2, num_threads: usize, + batch_size: usize, mut delta_sum: ArrayViewMut2, mut hit_count: ArrayViewMut2, ) where @@ -498,7 +559,11 @@ fn calc_feature_delta_sum_impl( .build() .expect("Cannot build rayon ThreadPool") .install(|| { - zip.par_for_each(inner_fn); + zip.into_par_iter().with_min_len(batch_size).for_each( + |(sample, delta_sum_row, hit_count_row)| { + inner_fn(sample, delta_sum_row, hit_count_row) + }, + ); }); } } diff --git a/src/coniferest/aadforest.py b/src/coniferest/aadforest.py index 9e88973..f276bc4 100644 --- a/src/coniferest/aadforest.py +++ b/src/coniferest/aadforest.py @@ -37,7 +37,14 @@ def score_samples(self, x, weights=None): if weights is None: weights = self.weights - return calc_paths_sum(self.selectors, self.node_offsets, x, weights, num_threads=self.num_threads) + return calc_paths_sum( + self.selectors, + self.node_offsets, + x, + weights, + num_threads=self.num_threads, + batch_size=self.get_batch_size(self.n_trees), + ) def loss( self, @@ -185,7 +192,12 @@ def __init__( map_value=None, ): super().__init__( - trees=[], n_subsamples=n_subsamples, max_depth=max_depth, n_jobs=n_jobs, random_seed=random_seed + trees=[], + n_subsamples=n_subsamples, + max_depth=max_depth, + n_jobs=n_jobs, + random_seed=random_seed, + sampletrees_per_batch=sampletrees_per_batch, ) self.n_trees = n_trees diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index d89c107..dd439fa 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -11,7 +11,7 @@ class ForestEvaluator: selector_dtype = selector_dtype - def __init__(self, samples, selectors, node_offsets, leaf_offsets, *, num_threads): + def __init__(self, samples, selectors, node_offsets, leaf_offsets, *, num_threads, sampletrees_per_batch): """ Base class for the forest evaluators. Does the trivial job: * runs calc_paths_sum written in Rust, @@ -42,10 +42,9 @@ def __init__(self, samples, selectors, node_offsets, leaf_offsets, *, num_thread self.node_offsets = node_offsets self.leaf_offsets = leaf_offsets - if num_threads is None or num_threads < 1: - # Count of available CPUs is not a simple thing, see loky's implementation here: - # https://github.com/joblib/joblib/blob/476ff8e62b221fc5816bad9b55dec8883d4f157c/joblib/externals/loky/backend/context.py#L83 - self.num_threads = joblib.cpu_count() + if num_threads is None or num_threads < 0: + # Ask Rust's rayon to use all available threads + self.num_threads = 0 else: self.num_threads = num_threads @@ -144,18 +143,30 @@ def score_samples(self, x): x = np.ascontiguousarray(x) return -( - 2 - ** ( - -calc_paths_sum(self.selectors, self.node_offsets, x, num_threads=self.num_threads) - / (self.average_path_length(self.samples) * self.n_trees) + 2 + ** ( + -calc_paths_sum( + self.selectors, + self.node_offsets, + x, + num_threads=self.num_threads, + batch_size=self.get_batch_size(self.n_trees), ) + / (self.average_path_length(self.samples) * self.n_trees) + ) ) def _feature_delta_sum(self, x): if not x.flags["C_CONTIGUOUS"]: x = np.ascontiguousarray(x) - return calc_feature_delta_sum(self.selectors, self.node_offsets, x, num_threads=self.num_threads) + return calc_feature_delta_sum( + self.selectors, + self.node_offsets, + x, + num_threads=self.num_threads, + batch_size=self.get_batch_size(self.n_trees), + ) def feature_signature(self, x): delta_sum, hit_count = self._feature_delta_sum(x) diff --git a/tests/test_aadforest.py b/tests/test_aadforest.py index f2a254d..331f364 100644 --- a/tests/test_aadforest.py +++ b/tests/test_aadforest.py @@ -41,8 +41,11 @@ def test_prior_influence_callable(): assert np.argmin(scores) == data.shape[0] - 1 +# Single-thread and parallel implementations are a bit different, so here we check both. +# We use n_thread parameter instead of n_jobs, which is a fixture in conftest.py +@pytest.mark.parametrize("n_thread", [1, 2]) @pytest.mark.regression -def test_regression_fit_known(regression_data): +def test_regression_fit_known(n_thread, regression_data): random_seed = 0 n_samples = 1024 n_features = 16 @@ -53,7 +56,8 @@ def test_regression_fit_known(regression_data): known_data = data[rng.choice(n_samples, n_known, replace=False)] known_labels = rng.choice([-1, 1], n_known, replace=True) - forest = AADForest(n_trees=n_trees, random_seed=random_seed) + # This small sampletrees_per_batch is inefficient, but it's good for testing to guarantee parallel execution. + forest = AADForest(n_trees=n_trees, random_seed=random_seed, n_jobs=n_thread, sampletrees_per_batch=2048) forest.fit(data) pre_fit_known_scores = forest.score_samples(data) From 0e55d8785045798b51031ffa53ed77f403ae1b66 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 28 May 2024 11:35:19 -0400 Subject: [PATCH 17/40] #[allow(clippy::too_many_arguments)] --- rust/src/tree_traversal.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 27e23e9..193e3fb 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -25,6 +25,7 @@ trait DataTrait<'py> { batch_size: usize, ) -> PyResult>>; + #[allow(clippy::too_many_arguments)] fn calc_paths_sum_transpose( &self, py: Python<'py>, @@ -387,6 +388,7 @@ fn calc_paths_sum_impl( } } +#[allow(clippy::too_many_arguments)] #[pyfunction] #[pyo3(signature = (selectors, node_offsets, leaf_offsets, data, weights = None, *, num_threads, batch_size))] pub(crate) fn calc_paths_sum_transpose<'py>( @@ -410,6 +412,7 @@ pub(crate) fn calc_paths_sum_transpose<'py>( ) } +#[allow(clippy::too_many_arguments)] fn calc_paths_sum_transpose_impl( selectors: ArrayView1, node_offsets: ArrayView1, From bbd52a75be34058b84748d3ed7515d79365c62eb Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 3 Sep 2024 13:31:33 -0400 Subject: [PATCH 18/40] Run cargo update --- rust/Cargo.lock | 123 +++++++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 58 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4e93336..fa70cbe 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4,15 +4,15 @@ version = 3 [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "bitflags" -version = "1.3.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "cfg-if" @@ -54,15 +54,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "either" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "enum_dispatch" @@ -84,9 +84,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "indoc" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "itertools" @@ -99,15 +99,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "matrixmultiply" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" dependencies = [ "autocfg", "rawpointer", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -196,9 +196,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", @@ -219,15 +219,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -312,9 +312,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -332,9 +332,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ "bitflags", ] @@ -353,15 +353,15 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "syn" -version = "2.0.52" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -370,9 +370,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "unicode-ident" @@ -388,13 +388,14 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", + "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", @@ -403,42 +404,48 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" From 84b15818772369329d02da55a7e4d5004edeae21 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 21:32:34 -0400 Subject: [PATCH 19/40] Implement calc_apply --- rust/src/lib.rs | 5 +- rust/src/tree_traversal.rs | 116 +++++++++++++++++++++++++++++++++++- src/coniferest/evaluator.py | 6 +- 3 files changed, 119 insertions(+), 8 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 4957bd8..d32c179 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -3,7 +3,9 @@ mod selector; mod tree_traversal; use crate::selector::Selector; -use crate::tree_traversal::{calc_feature_delta_sum, calc_paths_sum, calc_paths_sum_transpose}; +use crate::tree_traversal::{ + calc_apply, calc_feature_delta_sum, calc_paths_sum, calc_paths_sum_transpose, +}; use pyo3::prelude::*; #[pymodule] @@ -13,5 +15,6 @@ fn rust_module(py: Python, m: &Bound) -> PyResult<()> { m.add_function(wrap_pyfunction!(calc_paths_sum, m)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum_transpose, m)?)?; m.add_function(wrap_pyfunction!(calc_feature_delta_sum, m)?)?; + m.add_function(wrap_pyfunction!(calc_apply, m)?)?; Ok(()) } diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 193e3fb..4c8cc3e 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -45,6 +45,15 @@ trait DataTrait<'py> { num_threads: usize, batch_size: usize, ) -> PyResult>; + + fn calc_apply( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, + num_threads: usize, + batch_size: usize, + ) -> PyResult>>; } impl<'py, T> DataTrait<'py> for Bound<'py, PyArray2> @@ -201,6 +210,47 @@ where (delta_sum, hit_count) }) } + + fn calc_apply( + &self, + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, + num_threads: usize, + batch_size: usize, + ) -> PyResult>> { + let selectors = selectors.readonly(); + let selectors_view = selectors.as_array(); + check_selectors(selectors_view)?; + + let node_offsets = node_offsets.readonly(); + let node_offsets_view = node_offsets.as_array(); + check_node_offsets(node_offsets_view, selectors.len()?)?; + + let data = self.readonly(); + let data_view = data.as_array(); + check_data(data_view)?; + + let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; + + Ok({ + let leafs = + PyArray2::zeros_bound(py, (data_view.nrows(), node_offsets_view.len() - 1), false); + // SAFETY: this call invalidates other views, but it is the only view we need + let leafs_view = unsafe { leafs.as_array_mut() }; + + calc_apply_impl( + selectors_view, + node_offsets_view, + data_view, + num_threads, + batch_size, + leafs_view, + ); + + leafs + }) + } } #[enum_dispatch(DataTrait)] @@ -260,11 +310,12 @@ fn check_node_offsets(node_offsets: ArrayView1, selectors_length: usize) } } if node_offsets[node_offsets.len() - 1] > selectors_length { - return Err(PyValueError::new_err( + Err(PyValueError::new_err( "node_offsets are out of range of the selectors", - )); + )) + } else { + Ok(()) } - Ok(()) } else { Err(PyValueError::new_err( "node_offsets must be contiguous and in memory order", @@ -570,3 +621,62 @@ fn calc_feature_delta_sum_impl( }); } } + +#[pyfunction] +#[pyo3(signature = (selectors, node_offsets, data, *, num_threads, batch_size))] +pub(crate) fn calc_apply<'py>( + py: Python<'py>, + selectors: Bound<'py, PyArray1>, + node_offsets: Bound<'py, PyArray1>, + data: Data<'py>, + num_threads: usize, + batch_size: usize, +) -> PyResult>> { + data.calc_apply(py, selectors, node_offsets, num_threads, batch_size) +} + +fn calc_apply_impl( + selectors: ArrayView1, + node_offsets: ArrayView1, + data: ArrayView2, + num_threads: usize, + batch_size: usize, + mut leafs: ArrayViewMut2, +) where + T: Copy + Send + Sync + PartialOrd + 'static, + f64: AsPrimitive, +{ + let node_offsets = node_offsets.as_slice().unwrap(); + let selectors = selectors.as_slice().unwrap(); + + let inner_fn = |sample: ArrayView1, mut sample_leafs: ArrayViewMut1| { + let sample_slice = sample.as_slice().unwrap(); + let leafs_slice = sample_leafs.as_slice_mut().unwrap(); + for ((tree_start, tree_end), leaf_id) in node_offsets + .iter() + .copied() + .tuple_windows() + .zip(leafs_slice.iter_mut()) + { + let tree_selectors = unsafe { selectors.get_unchecked(tree_start..tree_end) }; + let leaf = find_leaf(tree_selectors, sample_slice); + *leaf_id = leaf.left; + } + }; + + let zip = Zip::from(data.rows()).and(leafs.rows_mut()); + + if num_threads == 1 { + zip.for_each(inner_fn); + } else { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + .expect("Cannot build rayon ThreadPool") + .install(|| { + zip.into_par_iter() + .with_min_len(batch_size) + .for_each(|(sample, sample_leafs)| inner_fn(sample, sample_leafs)); + }); + } +} diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index dd439fa..44e45a3 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -2,7 +2,7 @@ import numpy as np -from .calc_paths_sum import calc_feature_delta_sum, calc_paths_sum, selector_dtype # noqa +from .calc_paths_sum import calc_apply, calc_feature_delta_sum, calc_paths_sum, selector_dtype # noqa from .utils import average_path_length __all__ = ["ForestEvaluator"] @@ -103,7 +103,7 @@ def combine_selectors(cls, selectors_list): node_offsets[1:] = np.add.accumulate(lens) for i in range(len(selectors_list)): - selectors[node_offsets[i]: node_offsets[i + 1]] = selectors_list[i] + selectors[node_offsets[i] : node_offsets[i + 1]] = selectors_list[i] # Assign a unique sequential index to every leaf # The index is used for weighted scores @@ -179,8 +179,6 @@ def feature_importance(self, x): return np.sum(delta_sum, axis=0) / np.sum(hit_count, axis=0) / self.average_path_length(self.samples) def apply(self, x): - raise NotImplemented("Not implemented in Rust yet") - if not x.flags["C_CONTIGUOUS"]: x = np.ascontiguousarray(x) From 8f7a5989c1dae33889bed2cb37f8d561e8d919b5 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 21:38:06 -0400 Subject: [PATCH 20/40] Cargo clippy --- rust/src/mut_slices.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index c66814f..bebe9f2 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -12,7 +12,7 @@ impl<'sl, 'off, T> MutSlices<'sl, 'off, T> { } } -impl<'sl, 'off, T> Iterator for MutSlices<'sl, 'off, T> { +impl<'sl, T> Iterator for MutSlices<'sl, '_, T> { type Item = &'sl mut [T]; fn next(&mut self) -> Option { @@ -37,7 +37,7 @@ impl<'sl, 'off, T> Iterator for MutSlices<'sl, 'off, T> { } } -impl<'sl, 'off, T> DoubleEndedIterator for MutSlices<'sl, 'off, T> { +impl DoubleEndedIterator for MutSlices<'_, '_, T> { fn next_back(&mut self) -> Option { let offsets_len = self.offsets.len(); if offsets_len <= 1 { @@ -56,14 +56,14 @@ impl<'sl, 'off, T> DoubleEndedIterator for MutSlices<'sl, 'off, T> { } } -impl<'sl, 'off, T> ExactSizeIterator for MutSlices<'sl, 'off, T> { +impl ExactSizeIterator for MutSlices<'_, '_, T> { fn len(&self) -> usize { self.offsets.len() - 1 } } // Following rayon's ChunksMut implementation -impl<'sl, 'off, T> ParallelIterator for MutSlices<'sl, 'off, T> +impl<'sl, T> ParallelIterator for MutSlices<'sl, '_, T> where T: Send, { @@ -81,7 +81,7 @@ where } } -impl<'sl, 'off, T> IndexedParallelIterator for MutSlices<'sl, 'off, T> +impl IndexedParallelIterator for MutSlices<'_, '_, T> where T: Send, { From 7ee9d9ec8280256f6108e8b76a86054b73ad0bef Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:01:47 -0400 Subject: [PATCH 21/40] Rebane ext moduls to calc_trees --- pyproject.toml | 2 +- rust/Cargo.toml | 2 +- rust/src/lib.rs | 3 +-- rust/src/mut_slices.rs | 2 +- rust/src/selector.rs | 2 +- rust/src/tree_traversal.rs | 2 +- src/coniferest/evaluator.py | 2 +- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 13a91ea..00a0bda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dev = [ "Source Code" = "https://github.com/snad-space/coniferest" [tool.maturin] -module-name = "coniferest.calc_paths_sum" +module-name = "coniferest.calc_trees" # It asks to use Cargo.lock to make the build reproducible locked = true diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9762a24..e998943 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -4,7 +4,7 @@ version = "0.0.16" edition = "2021" [lib] -name = "coniferest" +name = "calc_trees" crate-type = ["cdylib"] # We'd like to build fast code with `pip install -e '.[dev]'` diff --git a/rust/src/lib.rs b/rust/src/lib.rs index d32c179..0e3e0db 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -9,8 +9,7 @@ use crate::tree_traversal::{ use pyo3::prelude::*; #[pymodule] -#[pyo3(name = "calc_paths_sum")] -fn rust_module(py: Python, m: &Bound) -> PyResult<()> { +fn calc_trees(py: Python, m: &Bound) -> PyResult<()> { m.add("selector_dtype", Selector::dtype(py)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum, m)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum_transpose, m)?)?; diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index bebe9f2..75b6906 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -1,4 +1,4 @@ -use rayon::iter::plumbing::{bridge, Consumer, Producer, ProducerCallback, UnindexedConsumer}; +use rayon::iter::plumbing::{Consumer, Producer, ProducerCallback, UnindexedConsumer, bridge}; use rayon::iter::{IndexedParallelIterator, ParallelIterator}; pub struct MutSlices<'sl, 'off, T> { diff --git a/rust/src/selector.rs b/rust/src/selector.rs index 850a909..c942821 100644 --- a/rust/src/selector.rs +++ b/rust/src/selector.rs @@ -2,7 +2,7 @@ use numpy::{Element, PyArrayDescr}; use pyo3::prelude::{PyAnyMethods, PyDictMethods}; use pyo3::sync::GILOnceCell; use pyo3::types::PyDict; -use pyo3::{py_run, Bound, Py, PyResult, Python}; +use pyo3::{Bound, Py, PyResult, Python, py_run}; static SELECTOR_DTYPE_CELL: GILOnceCell> = GILOnceCell::new(); diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 4c8cc3e..7fb5d8e 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -8,7 +8,7 @@ use num_traits::AsPrimitive; use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; -use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; +use pyo3::{Bound, FromPyObject, PyResult, Python, pyfunction}; use rayon::prelude::*; type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); diff --git a/src/coniferest/evaluator.py b/src/coniferest/evaluator.py index 44e45a3..e46fbf0 100644 --- a/src/coniferest/evaluator.py +++ b/src/coniferest/evaluator.py @@ -2,7 +2,7 @@ import numpy as np -from .calc_paths_sum import calc_apply, calc_feature_delta_sum, calc_paths_sum, selector_dtype # noqa +from .calc_trees import calc_apply, calc_feature_delta_sum, calc_paths_sum, selector_dtype # noqa from .utils import average_path_length __all__ = ["ForestEvaluator"] From b6071889b0fa231ac1a57c81670716a7b5c21114 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:13:49 -0400 Subject: [PATCH 22/40] pyO3 & numpy 0.22 --- rust/Cargo.lock | 215 ++++++++++--------------------------------- rust/Cargo.toml | 6 +- rust/src/selector.rs | 4 + 3 files changed, 58 insertions(+), 167 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index fa70cbe..c23a4a2 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,18 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - -[[package]] -name = "bitflags" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "cfg-if" @@ -35,9 +29,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -54,15 +48,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "either" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "enum_dispatch" @@ -78,15 +72,15 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indoc" -version = "2.0.5" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "itertools" @@ -99,19 +93,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" - -[[package]] -name = "lock_api" -version = "0.4.12" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "matrixmultiply" @@ -134,14 +118,16 @@ dependencies = [ [[package]] name = "ndarray" -version = "0.15.6" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" dependencies = [ "matrixmultiply", "num-complex", "num-integer", "num-traits", + "portable-atomic", + "portable-atomic-util", "rawpointer", "rayon", ] @@ -175,9 +161,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.21.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec170733ca37175f5d75a5bea5911d6ff45d2cd52849ce98b685394e4f2f37f4" +checksum = "edb929bc0da91a4d85ed6c0a84deaa53d411abfb387fc271124f91bf6b89f14e" dependencies = [ "libc", "ndarray", @@ -190,59 +176,45 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] -name = "parking_lot" -version = "0.12.3" +name = "portable-atomic" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] -name = "parking_lot_core" -version = "0.9.10" +name = "portable-atomic-util" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", + "portable-atomic", ] -[[package]] -name = "portable-atomic" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" - [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -252,9 +224,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" dependencies = [ "once_cell", "target-lexicon", @@ -262,9 +234,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" dependencies = [ "libc", "pyo3-build-config", @@ -272,9 +244,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -284,9 +256,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ "heck", "proc-macro2", @@ -297,9 +269,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -330,38 +302,17 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" -dependencies = [ - "bitflags", -] - [[package]] name = "rustc-hash" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - [[package]] name = "syn" -version = "2.0.77" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -376,76 +327,12 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unindent" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e998943..58c5d0c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,10 +22,10 @@ default = ["pyo3/abi3-py39"] [dependencies] enum_dispatch = "0.3" itertools = "0.12" -pyo3 = { version = "0.21", features = ["extension-module"] } +pyo3 = { version = "0.22", features = ["extension-module"] } # Needs to be consistent with ndarray dependecy in numpy -ndarray = { version = "0.15", features = ["rayon"] } +ndarray = { version = "0.16", features = ["rayon"] } num-traits = "0.2" -numpy = "0.21" +numpy = "0.22" # Needs to be consistent with rayon dependecy in ndarray rayon = "1.9" diff --git a/rust/src/selector.rs b/rust/src/selector.rs index c942821..302eaef 100644 --- a/rust/src/selector.rs +++ b/rust/src/selector.rs @@ -72,4 +72,8 @@ unsafe impl Element for Selector { fn get_dtype_bound(py: Python) -> Bound { Self::dtype(py).unwrap() } + + fn clone_ref(&self, py: Python<'_>) -> Self { + self.clone() + } } From 0db0ad48e0b8fd15c03c8e8bafa09d58303225fe Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:16:24 -0400 Subject: [PATCH 23/40] pyO3 & numpy 0.23 --- rust/Cargo.lock | 28 ++++++++++++++-------------- rust/Cargo.toml | 4 ++-- rust/src/selector.rs | 6 +++--- rust/src/tree_traversal.rs | 12 +++++------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c23a4a2..deca497 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -161,9 +161,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.22.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb929bc0da91a4d85ed6c0a84deaa53d411abfb387fc271124f91bf6b89f14e" +checksum = "b94caae805f998a07d33af06e6a3891e38556051b8045c615470a71590e13e78" dependencies = [ "libc", "ndarray", @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" dependencies = [ "cfg-if", "indoc", @@ -224,9 +224,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" dependencies = [ "once_cell", "target-lexicon", @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" dependencies = [ "libc", "pyo3-build-config", @@ -244,9 +244,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" dependencies = [ "heck", "proc-macro2", @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "syn" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 58c5d0c..54c26c0 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,10 +22,10 @@ default = ["pyo3/abi3-py39"] [dependencies] enum_dispatch = "0.3" itertools = "0.12" -pyo3 = { version = "0.22", features = ["extension-module"] } +pyo3 = { version = "0.23", features = ["extension-module"] } # Needs to be consistent with ndarray dependecy in numpy ndarray = { version = "0.16", features = ["rayon"] } num-traits = "0.2" -numpy = "0.22" +numpy = "0.23" # Needs to be consistent with rayon dependecy in ndarray rayon = "1.9" diff --git a/rust/src/selector.rs b/rust/src/selector.rs index 302eaef..bf28628 100644 --- a/rust/src/selector.rs +++ b/rust/src/selector.rs @@ -29,7 +29,7 @@ impl Selector { pub(crate) fn dtype(py: Python) -> PyResult> { let unbind_dtype = SELECTOR_DTYPE_CELL.get_or_try_init(py, || -> PyResult<_> { - let locals = PyDict::new_bound(py); + let locals = PyDict::new(py); py_run!( py, *locals, @@ -69,11 +69,11 @@ impl Selector { unsafe impl Element for Selector { const IS_COPY: bool = true; - fn get_dtype_bound(py: Python) -> Bound { + fn get_dtype(py: Python) -> Bound { Self::dtype(py).unwrap() } - fn clone_ref(&self, py: Python<'_>) -> Self { + fn clone_ref(&self, _py: Python<'_>) -> Self { self.clone() } } diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 7fb5d8e..05bd417 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -88,7 +88,7 @@ where let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; Ok({ - let paths = PyArray1::zeros_bound(py, data_view.nrows(), false); + let paths = PyArray1::zeros(py, data_view.nrows(), false); // SAFETY: this call invalidates other views, but it is the only view we need let paths_view_mut = unsafe { paths.as_array_mut() }; @@ -138,7 +138,7 @@ where let num_threads = get_num_threads(data_view.ncols(), num_threads, batch_size)?; Ok({ - let values = PyArray1::zeros_bound( + let values = PyArray1::zeros( py, *leaf_offsets_view .last() @@ -187,10 +187,8 @@ where let num_threads = get_num_threads(data_view.nrows(), num_threads, batch_size)?; Ok({ - let delta_sum = - PyArray2::zeros_bound(py, (data_view.nrows(), data_view.ncols()), false); - let hit_count = - PyArray2::zeros_bound(py, (data_view.nrows(), data_view.ncols()), false); + let delta_sum = PyArray2::zeros(py, (data_view.nrows(), data_view.ncols()), false); + let hit_count = PyArray2::zeros(py, (data_view.nrows(), data_view.ncols()), false); // SAFETY: this call invalidates other views, but it is the only view we need let delta_sum_view = unsafe { delta_sum.as_array_mut() }; @@ -235,7 +233,7 @@ where Ok({ let leafs = - PyArray2::zeros_bound(py, (data_view.nrows(), node_offsets_view.len() - 1), false); + PyArray2::zeros(py, (data_view.nrows(), node_offsets_view.len() - 1), false); // SAFETY: this call invalidates other views, but it is the only view we need let leafs_view = unsafe { leafs.as_array_mut() }; From 90356913373f9399ab5874fbcdbe2b02d83eb9b0 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:17:41 -0400 Subject: [PATCH 24/40] Update rust deps --- rust/Cargo.lock | 33 +++++++++++++++++---------------- rust/Cargo.toml | 8 ++++---- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index deca497..18941bb 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -84,9 +84,9 @@ checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "itertools" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -161,9 +161,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94caae805f998a07d33af06e6a3891e38556051b8045c615470a71590e13e78" +checksum = "a7cfbf3f0feededcaa4d289fe3079b03659e85c5b5a177f4ba6fb01ab4fb3e39" dependencies = [ "libc", "ndarray", @@ -171,6 +171,7 @@ dependencies = [ "num-integer", "num-traits", "pyo3", + "pyo3-build-config", "rustc-hash", ] @@ -206,9 +207,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" dependencies = [ "cfg-if", "indoc", @@ -224,9 +225,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" dependencies = [ "once_cell", "target-lexicon", @@ -234,9 +235,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" dependencies = [ "libc", "pyo3-build-config", @@ -244,9 +245,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -256,9 +257,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" dependencies = [ "heck", "proc-macro2", @@ -321,9 +322,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "unicode-ident" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 54c26c0..eee80f8 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,11 +21,11 @@ default = ["pyo3/abi3-py39"] [dependencies] enum_dispatch = "0.3" -itertools = "0.12" -pyo3 = { version = "0.23", features = ["extension-module"] } +itertools = "0.14" +pyo3 = { version = "0.24", features = ["extension-module"] } # Needs to be consistent with ndarray dependecy in numpy ndarray = { version = "0.16", features = ["rayon"] } num-traits = "0.2" -numpy = "0.23" +numpy = "0.24" # Needs to be consistent with rayon dependecy in ndarray -rayon = "1.9" +rayon = "1.10" From 6cae3dd1bd6319d44cb508af026eb9614c7833f2 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:21:20 -0400 Subject: [PATCH 25/40] Support free-threading CPython --- rust/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 0e3e0db..23df59e 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -8,7 +8,7 @@ use crate::tree_traversal::{ }; use pyo3::prelude::*; -#[pymodule] +#[pymodule(gil_used = false)] fn calc_trees(py: Python, m: &Bound) -> PyResult<()> { m.add("selector_dtype", Selector::dtype(py)?)?; m.add_function(wrap_pyfunction!(calc_paths_sum, m)?)?; From 98f37ca71de5fcdb315e4c1090e8b3192be1a38a Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:23:12 -0400 Subject: [PATCH 26/40] Fix rust tests --- rust/src/mut_slices.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index 75b6906..04027b5 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -193,7 +193,7 @@ mod tests { let sum: usize = ParallelIterator::map(slices, |slice| slice.iter().sum::()).sum(); - assert_eq!(sum, data.iter().sum()); + assert_eq!(sum, data.iter().sum::()); } #[test] From a022f8e298970deb8ea0f89fb88698a4b9eb77a4 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 8 May 2025 22:23:44 -0400 Subject: [PATCH 27/40] clippy --- rust/src/selector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/selector.rs b/rust/src/selector.rs index bf28628..2b2df20 100644 --- a/rust/src/selector.rs +++ b/rust/src/selector.rs @@ -74,6 +74,6 @@ unsafe impl Element for Selector { } fn clone_ref(&self, _py: Python<'_>) -> Self { - self.clone() + *self } } From 354260ed2b47625c6f89241c1ac2a67c7e60ee71 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Fri, 9 May 2025 11:31:26 -0400 Subject: [PATCH 28/40] Set version via Cargo.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 00a0bda..12223c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,6 @@ build-backend = "maturin" [project] name = "coniferest" -version = "0.0.14" description = "Coniferous forests for better machine learning" readme = "README.md" requires-python = ">=3.10" @@ -29,6 +28,7 @@ dependencies = [ "scikit-learn>=1.4,<2", "onnxconverter-common", ] +dynamic = ["version"] [project.optional-dependencies] datasets = [ From cf2789c3ab96de7437ef1a0ccdfcd8ced49f93df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 16:14:13 +0000 Subject: [PATCH 29/40] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rust/src/mut_slices.rs | 2 +- rust/src/selector.rs | 2 +- rust/src/tree_traversal.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index 04027b5..d7ebaed 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -1,4 +1,4 @@ -use rayon::iter::plumbing::{Consumer, Producer, ProducerCallback, UnindexedConsumer, bridge}; +use rayon::iter::plumbing::{bridge, Consumer, Producer, ProducerCallback, UnindexedConsumer}; use rayon::iter::{IndexedParallelIterator, ParallelIterator}; pub struct MutSlices<'sl, 'off, T> { diff --git a/rust/src/selector.rs b/rust/src/selector.rs index 2b2df20..498f12d 100644 --- a/rust/src/selector.rs +++ b/rust/src/selector.rs @@ -2,7 +2,7 @@ use numpy::{Element, PyArrayDescr}; use pyo3::prelude::{PyAnyMethods, PyDictMethods}; use pyo3::sync::GILOnceCell; use pyo3::types::PyDict; -use pyo3::{Bound, Py, PyResult, Python, py_run}; +use pyo3::{py_run, Bound, Py, PyResult, Python}; static SELECTOR_DTYPE_CELL: GILOnceCell> = GILOnceCell::new(); diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 05bd417..2180d24 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -8,7 +8,7 @@ use num_traits::AsPrimitive; use numpy::{Element, PyArray1, PyArray2, PyArrayMethods}; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; -use pyo3::{Bound, FromPyObject, PyResult, Python, pyfunction}; +use pyo3::{pyfunction, Bound, FromPyObject, PyResult, Python}; use rayon::prelude::*; type DeltaSumHitCount<'py> = (Bound<'py, PyArray2>, Bound<'py, PyArray2>); From 8c4f0b16968bee13b2ec6b17ee7c7b7c355210e8 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Fri, 16 May 2025 11:34:02 -0400 Subject: [PATCH 30/40] MutSlices::next(_back) changes --- rust/src/mut_slices.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rust/src/mut_slices.rs b/rust/src/mut_slices.rs index d7ebaed..a42c4e2 100644 --- a/rust/src/mut_slices.rs +++ b/rust/src/mut_slices.rs @@ -20,10 +20,11 @@ impl<'sl, T> Iterator for MutSlices<'sl, '_, T> { return None; } - // Split slice, save right. Here we temporarily replace slice with an empty one - let (left, right) = + // Split slice, save right back. + // take() temporarily replaces self.slice with an empty slice + let left: &mut [T]; + (left, self.slice) = std::mem::take(&mut self.slice).split_at_mut(self.offsets[1] - self.offsets[0]); - self.slice = right; // Move offsets to the right self.offsets = &self.offsets[1..]; @@ -44,10 +45,11 @@ impl DoubleEndedIterator for MutSlices<'_, '_, T> { return None; } - // Split slice, save left. Here we temporarily replace slice with an empty one - let (left, right) = std::mem::take(&mut self.slice) + // Split slice, save left back. + // take() temporarily replaces self.slice with an empty slice + let right: &mut [T]; + (self.slice, right) = std::mem::take(&mut self.slice) .split_at_mut(self.offsets[offsets_len - 2] - self.offsets[0]); - self.slice = left; // Move offsets to the left self.offsets = &self.offsets[..offsets_len - 1]; From d987cd748c130f9ad3f91efc95752f816f471344 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Tue, 1 Jul 2025 17:17:10 -0400 Subject: [PATCH 31/40] Bump ABI3 to py3.10+ --- rust/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index eee80f8..d6b7c45 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -17,7 +17,7 @@ lto = true codegen-units = 1 [features] -default = ["pyo3/abi3-py39"] +default = ["pyo3/abi3-py310"] [dependencies] enum_dispatch = "0.3" From ea546501873a96a85bf327bfc6519fa80ea51c5b Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 2 Jul 2025 08:23:58 -0400 Subject: [PATCH 32/40] cargo update --- rust/Cargo.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 18941bb..12bfbc8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4,19 +4,19 @@ version = 4 [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "coniferest" -version = "0.0.14" +version = "0.0.16" dependencies = [ "enum_dispatch", "itertools", @@ -93,15 +93,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "matrixmultiply" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" dependencies = [ "autocfg", "rawpointer", @@ -183,9 +183,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "portable-atomic-util" @@ -311,9 +311,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "syn" -version = "2.0.101" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", From 81d8f4f4cbf9accd841e2c45030a6f0bf3417110 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Wed, 2 Jul 2025 17:09:34 -0400 Subject: [PATCH 33/40] Bumpy pyo3/numpy to 0.25 --- rust/Cargo.lock | 31 ++++++++++++------------------- rust/Cargo.toml | 4 ++-- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 12bfbc8..5b14187 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -8,12 +8,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "cfg-if" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" - [[package]] name = "coniferest" version = "0.0.16" @@ -161,9 +155,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cfbf3f0feededcaa4d289fe3079b03659e85c5b5a177f4ba6fb01ab4fb3e39" +checksum = "29f1dee9aa8d3f6f8e8b9af3803006101bb3653866ef056d530d53ae68587191" dependencies = [ "libc", "ndarray", @@ -207,11 +201,10 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", @@ -225,9 +218,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" dependencies = [ "once_cell", "target-lexicon", @@ -235,9 +228,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" dependencies = [ "libc", "pyo3-build-config", @@ -245,9 +238,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -257,9 +250,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" dependencies = [ "heck", "proc-macro2", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index d6b7c45..98d8153 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,10 +22,10 @@ default = ["pyo3/abi3-py310"] [dependencies] enum_dispatch = "0.3" itertools = "0.14" -pyo3 = { version = "0.24", features = ["extension-module"] } +pyo3 = { version = "0.25", features = ["extension-module"] } # Needs to be consistent with ndarray dependecy in numpy ndarray = { version = "0.16", features = ["rayon"] } num-traits = "0.2" -numpy = "0.24" +numpy = "0.25" # Needs to be consistent with rayon dependecy in ndarray rayon = "1.10" From 0229b7b2afd9d9fb34747400e1190bbb115cbd6a Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Mon, 13 May 2024 22:59:40 +0200 Subject: [PATCH 34/40] WIP: devnet notebook --- docs/notebooks/devnet.ipynb | 228 +++++++++++++++++++++++++++ docs/notebooks/devnet_datasets.ipynb | 200 +++++++++++++++++++++++ 2 files changed, 428 insertions(+) create mode 100644 docs/notebooks/devnet.ipynb create mode 100644 docs/notebooks/devnet_datasets.ipynb diff --git a/docs/notebooks/devnet.ipynb b/docs/notebooks/devnet.ipynb new file mode 100644 index 0000000..dcf36d2 --- /dev/null +++ b/docs/notebooks/devnet.ipynb @@ -0,0 +1,228 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, +<<<<<<< HEAD + "id": "ea4ae65a-d555-4b54-96f9-11eed006adc2", + "metadata": {}, + "outputs": [], + "source": [ + "# %pip uninstall -y coniferest\n", + "# %pip install 'git+https://github.com/snad-space/coniferest@fix-devent-celeba'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, +======= +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + "id": "3d9577061e9494ed", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T15:41:49.204695Z", + "start_time": "2024-03-13T15:41:49.201344Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "from coniferest.aadforest import AADForest\n", + "from coniferest.datasets import Dataset, DevNetDataset\n", + "from coniferest.isoforest import IsolationForest\n", + "from coniferest.label import Label\n", + "from coniferest.pineforest import PineForest\n", + "from coniferest.session.oracle import OracleSession, create_oracle_session" + ] + }, + { + "cell_type": "code", +<<<<<<< HEAD + "execution_count": 3, +======= + "execution_count": 2, +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + "id": "initial_id", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T15:41:49.210919Z", + "start_time": "2024-03-13T15:41:49.206277Z" + } + }, + "outputs": [], + "source": [ + "class Compare:\n", + " def __init__(self, dataset: Dataset, *, n_jobs=-1):\n", + " model_kwargs = {\n", + " 'n_trees': 128,\n", + " 'random_seed': 0,\n", + " 'n_jobs': n_jobs,\n", + " }\n", + " session_kwargs = {\n", + " 'data': dataset.data,\n", + " 'labels': dataset.labels,\n", + " 'max_iterations': 100,\n", + " }\n", + " \n", + " self.isoforest_session = create_oracle_session(\n", + " model=IsolationForest(**model_kwargs),\n", + " **session_kwargs,\n", + " )\n", + " self.aadforest_session = create_oracle_session(\n", + " model=AADForest(**model_kwargs),\n", + " **session_kwargs,\n", + " )\n", + " self.pineforest_session = create_oracle_session(\n", + " model=PineForest(**model_kwargs), #weight_ratio=4.0),\n", + " **session_kwargs,\n", + " )\n", + " \n", + " def run(self):\n", + " print(\"Running Isolation Forest\")\n", + " self.isoforest_session.run()\n", + " print(\"Running AAD Isolation Forest\")\n", + " self.aadforest_session.run()\n", + " print(\"Running Pine Forest\")\n", + " self.pineforest_session.run()\n", + " \n", + " return self\n", + " \n", +<<<<<<< HEAD + " def plot(self, dataset_name, savefig=False):\n", + " plt.figure(figsize=(8, 6))\n", + " plt.title(f'Dataset: {dataset_name}')\n", +======= + " def plot(self, title=None):\n", + " plt.figure(figsize=(8, 6))\n", + " if title is None:\n", + " title = 'AD performance curves'\n", + " plt.title(title)\n", +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + " \n", + " def performance(session):\n", + " return np.cumsum(np.array(list(session.known_labels.values())) == Label.A)\n", + "\n", +<<<<<<< HEAD + " plt.plot(performance(self.isoforest_session), label='Isolation Forest')\n", + " plt.plot(performance(self.aadforest_session), label='AAD Isolation Forest')\n", + " plt.plot(performance(self.pineforest_session), label='Pine Forest')\n", +======= + " plt.plot(performance(self.pineforest_session), label='Pine Forest')\n", + " plt.plot(performance(self.aadforest_session), label='AAD Isolation Forest')\n", + " plt.plot(performance(self.isoforest_session), label='Isolation Forest')\n", +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + " #plt.axhline(sum(self.dataset.labels == Label.A), color='grey')\n", + " plt.xlabel('number of iteration')\n", + " plt.ylabel('true anomalies detected')\n", + " plt.grid()\n", + " plt.legend()\n", +<<<<<<< HEAD + " if savefig:\n", + " plt.savefig(f'{dataset}.pdf')\n", +======= +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + " \n", + " return self" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71c337b3577915d5", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "donors\n", +<<<<<<< HEAD + "Running Isolation Forest\n", + "Running AAD Isolation Forest\n", + "Running Pine Forest\n", + "CPU times: user 40min 36s, sys: 8.35 s, total: 40min 44s\n", + "Wall time: 7min 40s\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq4AAAIjCAYAAADC0ZkAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACweklEQVR4nOzdd1QUZxfH8e/SOyiCgBU79p4YjRVrYjeaWKLGEnuLNcWWGKOxx96wd01i8tp7jWLvHQVFxIIgfdmd94+NGxELqywLy/2cw9GZnZ25y1Aus888P5WiKApCCCGEEEJkcBamLkAIIYQQQojUkMZVCCGEEEJkCtK4CiGEEEKITEEaVyGEEEIIkSlI4yqEEEIIITIFaVyFEEIIIUSmII2rEEIIIYTIFKRxFUIIIYQQmYI0rkIIIYQQIlOQxlUIIbKgJUuWoFKpuH37tqlLEUKIVJPGVQiRqTxvuJ5/2NnZ4ePjQ/369ZkxYwbPnj17530fOXKE0aNH8/Tp07Qr+D3Mnj2bJUuWmLoMIYTIMKRxFUJkSmPHjmX58uXMmTOHvn37AjBgwABKlSrFuXPn3mmfR44cYcyYMdK4CiFEBmVl6gKEEOJdNGzYkIoVK+qXR4wYwZ49e/j0009p0qQJly9fxt7e3oQViveRlJSEVqvFxsbG1KUIITIQueIqhDAbtWvX5ocffuDOnTusWLFCv/7cuXN06tSJAgUKYGdnh5eXF1999RWPHz/WbzN69GiGDBkCgK+vr34owvMxoAEBAdSuXRtPT09sbW0pXrw4c+bMSVHDiRMnqF+/Pjly5MDe3h5fX1+++uqrZNtotVqmTZtGiRIlsLOzI2fOnHz99ddERETot8mfPz8XL15k//79+lpq1qypf/zmzZvcvHkzVZ+XixcvUrt2bezt7cmdOzc//fQTWq32ldvOnj2bEiVKYGtri4+PD717905xBbpmzZqULFmSS5cuUatWLRwcHMiVKxcTJ05Msb/w8HC6dOlCzpw5sbOzo0yZMixdujTZNrdv30alUjFp0iSmTZtGwYIFsbW15dKlSwD89ttvlChRAgcHB7Jly0bFihVZtWpVql67EMK8yBVXIYRZ6dChA99++y07duygW7duAOzcuZNbt27RuXNnvLy8uHjxIvPnz+fixYv8888/qFQqWrRowbVr11i9ejVTp04lR44cAHh4eAAwZ84cSpQoQZMmTbCysuKvv/6iV69eaLVaevfuDeiatHr16uHh4cHw4cNxc3Pj9u3bbNq0KVmNX3/9NUuWLKFz587069ePoKAgZs6cyenTpzl8+DDW1tZMmzaNvn374uTkxHfffQdAzpw59fuoU6cOwFtvrgoLC6NWrVokJSUxfPhwHB0dmT9//iuvRo8ePZoxY8bg7+9Pz549uXr1KnPmzCEwMFBf13MRERE0aNCAFi1a0Lp1azZs2MCwYcMoVaoUDRs2BCAuLo6aNWty48YN+vTpg6+vL+vXr6dTp048ffqU/v37Jzt+QEAA8fHxdO/eHVtbW7Jnz86CBQvo168frVq1on///sTHx3Pu3DmOHTtG27Zt3/zFIIQwP4oQQmQiAQEBCqAEBga+dhtXV1elXLly+uXY2NgU26xevVoBlAMHDujX/frrrwqgBAUFpdj+VfuoX7++UqBAAf3y77///tbaDh48qADKypUrk63ftm1bivUlSpRQatSo8cr95MuXT8mXL99rj/PcgAEDFEA5duyYfl14eLji6uqa7LWGh4crNjY2Sr169RSNRqPfdubMmQqgLF68WL+uRo0aCqAsW7ZMvy4hIUHx8vJSWrZsqV83bdo0BVBWrFihX5eYmKhUqVJFcXJyUqKiohRFUZSgoCAFUFxcXJTw8PBk9Tdt2lQpUaLEW1+nECJrkKECQgiz4+TklGx2gRevLsbHx/Po0SM+/PBDAE6dOpWqfb64j8jISB49ekSNGjW4desWkZGRALi5uQHw999/o1arX7mf9evX4+rqSt26dXn06JH+o0KFCjg5ObF3795U1XP79u1UTWW1ZcsWPvzwQypXrqxf5+HhQbt27ZJtt2vXLhITExkwYAAWFv/9aujWrRsuLi7873//S7a9k5MT7du31y/b2NhQuXJlbt26lezYXl5efPHFF/p11tbW9OvXj+joaPbv359sny1bttRf4X7Ozc2Nu3fvEhgY+NbXKoQwf9K4CiHMTnR0NM7OzvrlJ0+e0L9/f3LmzIm9vT0eHh74+voC6JvOtzl8+DD+/v44Ojri5uaGh4cH3377bbJ91KhRg5YtWzJmzBhy5MhB06ZNCQgIICEhQb+f69evExkZiaenJx4eHsk+oqOjCQ8PT6tPAwB37tyhcOHCKdYXLVo0xXavWm9jY0OBAgX0jz+XO3duVCpVsnXZsmVLNk73+bFfbIQB/Pz8kh3zuefn5EXDhg3DycmJypUrU7hwYXr37s3hw4df+VqFEOZPxrgKIczK3bt3iYyMpFChQvp1rVu35siRIwwZMoSyZcvi5OSEVqulQYMGr71J6UU3b96kTp06FCtWjClTppAnTx5sbGzYsmULU6dO1e9DpVKxYcMG/vnnH/766y+2b9/OV199xeTJk/nnn3/0x/X09GTlypWvPNbLVxwzKktLy1euVxTlnff5qnG3fn5+XL16lb///ptt27axceNGZs+ezciRIxkzZsw7H0sIkTlJ4yqEMCvLly8HoH79+oDuJqLdu3czZswYRo4cqd/u+vXrKZ778hXE5/766y8SEhLYvHkzefPm1a9/3dv6H374IR9++CHjxo1j1apVtGvXjjVr1tC1a1cKFizIrl27qFq16lun63pdPYbIly/fK1/r1atXU2z3fH2BAgX06xMTEwkKCsLf3/+djn3u3Dm0Wm2yq65XrlxJdsy3cXR0pE2bNrRp04bExERatGjBuHHjGDFiBHZ2dgbXJYTIvGSogBDCbOzZs4cff/wRX19f/RjO51cGX74SOG3atBTPd3R0BEgx/dOr9hEZGUlAQECy7SIiIlIcp2zZsgD64QKtW7dGo9Hw448/pjh+UlJSsmM7Ojq+NgwhtdNhNWrUiH/++Yfjx4/r1z18+DDFFV9/f39sbGyYMWNGstewaNEiIiMj+eSTT956rFcdOywsjLVr1+rXJSUl8dtvv+Hk5ESNGjXeuo8XpywD3dCF4sWLoyjKa8cRCyHMl1xxFUJkSlu3buXKlSskJSXx4MED9uzZw86dO8mXLx+bN2/WX4lzcXGhevXqTJw4EbVaTa5cudixYwdBQUEp9lmhQgUAvvvuOz7//HOsra1p3Lgx9erVw8bGhsaNG/P1118THR3NggUL8PT05P79+/rnL126lNmzZ9O8eXMKFizIs2fPWLBgAS4uLjRq1AjQjYP9+uuvGT9+PGfOnKFevXpYW1tz/fp11q9fz/Tp02nVqpW+njlz5vDTTz9RqFAhPD09qV27NpD66bCGDh3K8uXLadCgAf3799dPh/X8auhzHh4ejBgxgjFjxtCgQQOaNGnC1atXmT17NpUqVUp2I1Zqde/enXnz5tGpUydOnjxJ/vz52bBhA4cPH2batGnJxiG/Tr169fDy8qJq1arkzJmTy5cvM3PmTD755JNUPV8IYWZMOaWBEEIY6vl0WM8/bGxsFC8vL6Vu3brK9OnT9VMsveju3btK8+bNFTc3N8XV1VX57LPPlNDQUAVQRo0alWzbH3/8UcmVK5diYWGRbLqozZs3K6VLl1bs7OyU/PnzKxMmTFAWL16cbJtTp04pX3zxhZI3b17F1tZW8fT0VD799FPlxIkTKWqaP3++UqFCBcXe3l5xdnZWSpUqpQwdOlQJDQ3VbxMWFqZ88sknirOzswIkmxortdNhKYqinDt3TqlRo4ZiZ2en5MqVS/nxxx+VRYsWvXLqr5kzZyrFihVTrK2tlZw5cyo9e/ZUIiIikm1To0aNV05R1bFjxxQ1PXjwQOncubOSI0cOxcbGRilVqpQSEBCQbJvn02H9+uuvKfY5b948pXr16oq7u7tia2urFCxYUBkyZIgSGRmZqtcuhDAvKkV5j5H0QgghhBBCpBMZ4yqEEEIIITIFaVyFEEIIIUSmII2rEEIIIYTIFKRxFUIIIYQQmYI0rkIIIYQQIlOQxlUIIYQQQmQKZh9AoNVqCQ0NxdnZOU3iE4UQQgghRNpSFIVnz57h4+OTLCL6ZWbfuIaGhpInTx5TlyGEEEIIId4iJCSE3Llzv/Zxs29cn0cChoSE4OLiYvTjqdVqduzYoY9xFJmTnEfzIOfRPMh5NA9yHjM/Y57DqKgo8uTJ89YoZ7NvXJ8PD3BxcUm3xtXBwQEXFxf5xszE5DyaBzmP5kHOo3mQ85j5pcc5fNuwTrk5SwghhBBCZArSuAohhBBCiExBGlchhBBCCJEpmP0Y19RQFIWkpCQ0Gs1770utVmNlZUV8fHya7E+YhrmeR0tLS6ysrGRqOCGEEJlSlm9cExMTuX//PrGxsWmyP0VR8PLyIiQkRJqDTMycz6ODgwPe3t7Y2NiYuhQhhBDCIFm6cdVqtQQFBWFpaYmPjw82Njbv3aRotVqio6NxcnJ64wS6ImMzx/OoKAqJiYk8fPiQoKAgChcubDavTQghRNaQpRvXxMREtFotefLkwcHBIU32qdVqSUxMxM7OTpqCTMxcz6O9vT3W1tbcuXNH//qEEEKIzMJ8fiO/B3NqTIR4G/l6F0IIkVnJbzAhhBBCCJEpSOMqhBBCCCEyBWlcs7jbt2+jUqk4c+ZMhtiPEEIIIcTrSOOaCXXq1IlmzZplqOPnyZOH+/fvU7JkSaMee/To0ahUqhQfu3btMupx31ZT2bJlTXZ8IYQQIqvI0rMKiLRjaWmJl5dXuhyrRIkSKRrV7Nmzv9O+EhMTZT5TIYQQIpOQK64vUBSF2MSk9/6IS9QY/BxFUd657g0bNlCqVCns7e1xd3fH39+fmJgYQDet09ixY8mdOze2traULVuWbdu2vXZfGo2GLl264Ovri729PUWLFmX69On6x0ePHs3SpUv5888/9Vc79+3b98qhAvv376dy5crY2tri7e3N8OHDSUpK0j9es2ZN+vXrx9ChQ8mePTteXl6MHj36ra/XysoKLy+vZB/Pm8/z589Tu3Zt/eeie/fuREdH65/7/GrxuHHj8PHxoWjRogCEhITQunVr3NzcyJ49O82aNSM4OFj/vH379lG5cmUcHR1xc3OjatWq3LlzhyVLljBmzBjOnj2r/3wsWbLkra9BCCGEEIaTK64viFNrKD5yu0mOfWlsfRxsDD8d9+/f54svvmDixIk0b96cZ8+ecfDgQX0jPH36dCZPnsy8efMoV64cixcvpkmTJly8eJHChQun2J9WqyV37tysX78ed3d3jhw5Qvfu3fH29qZ169YMHjyYy5cvExUVRUBAAKC72hkaGppsP/fu3aNRo0Z06tSJZcuWceXKFbp164adnV2y5nTp0qUMGjSIY8eOcfToUTp16kTVqlWpW7euwZ+LmJgY6tevT5UqVQgMDCQ8PJyuXbvSp0+fZM3k7t27cXFxYefOnYAu3vX58w4ePIiVlRU//vgjrVq14ty5c1hZWdGsWTO6devG6tWrSUxM5Pjx46hUKtq0acOFCxfYtm2b/iqwq6urwbULIYQQ4u2kcc3k7t+/T1JSEi1atCBfvnwAlCpVSv/4pEmTGDZsGJ9//jkAEyZMYO/evUybNo1Zs2al2J+1tTVjxozRL/v6+nL06FHWrVtH69atcXJywt7enoSEhDcODZg9ezZ58uRh5syZqFQqihUrRmhoKMOGDWPkyJH6uURLly7NqFGjAChcuDAzZ85k9+7db2xcz58/j5OTk365ePHiHD9+nFWrVhEfH8+yZctwdHQEYObMmTRu3JgJEyaQM2dOABwdHVm4cKH+Ku2KFSvQarUsXLhQn5y2ePFismfPrr/SGhkZyaeffkrBggUB8PPz0x/fyclJfxVYCCGEEMYjjesL7K0tuTS2/nvtQ6vV8izqGc4uzgZN9G5vbflOxytTpgx16tShVKlS1K9fn3r16tGqVSuyZctGVFQUoaGhVK1aNdlzqlatytmzZ1+7z1mzZrF48WKCg4OJi4sjMTHR4JuPLl++TJUqVZJF6FatWpXo6Gju3r1L3rx5AV3j+iJvb2/Cw8PfuO+iRYuyefNm/bKtra3+mGXKlNE3rc+PqdVquXr1qr5xLVWqVLJxrWfPnuXGjRs4OzsnO058fDw3b96kQYMGdOrUifr161O3bl38/f1p3bo13t7ehnxKhBBCiExBURSWXVqGfz5/cjnlMnU5yUjj+gKVSvVOb9e/SKvVkmRjiYONVbokFFlaWrJz506OHDnCjh07+O233/juu+84duwY7u7uBu9vzZo1DB48mMmTJ1OlShWcnZ359ddfOXbsmBGq113hfZFKpUKr1b7xOTY2NhQqVOidj/liYwsQHR1NhQoVWLlypX6dVqslOjoaX19fAAICAujXrx/btm1j7dq1fP/99+zcuZMPP/zwnesQQgghMhqNVsP44+NZe3UtG65tYF3jddhb2Zu6LD25OcsMqFQqqlatypgxYzh9+jQ2Njb8/vvvuLi44OPjw+HDh5Ntf/jwYYoXL/7KfR0+fJiPPvqIXr16Ua5cOQoVKsTNmzeTbWNjY4NGo3ljTX5+fhw9ejTZTWeHDx/G2dmZ3Llzv+MrfTM/Pz/Onj2rvzHt+TEtLCz0N2G9Svny5bl+/Tqenp4UKlRI/1GgQIFk41XLlSvHiBEjOHLkCCVLlmTVqlVA6j4fQgghREaXoElg8P7BrL26FhUqvij2RYZqWkEa10zv2LFj/Pzzz5w4cYLg4GA2bdrEw4cP9WMwhwwZwoQJE1i7di1Xr15l+PDhnDlzhv79+79yf4ULF+bEiRNs376da9eu8cMPPxAYGJhsm/z583Pu3DmuXr3Ko0ePUKvVKfbTq1cvQkJC6Nu3L1euXOHPP/9k1KhRDBo0yGhXotu1a4ednR0dO3bkwoUL7N27l759+9KhQwf9MIHXPS9Hjhw0bdqUgwcPEhQUxL59+xg2bBh3794lKCiIESNGcPToUe7cucOOHTu4fv26/nOcP39+goKCOHPmDI8ePSIhIcEor08IIYQwlqjEKL7e+TW7gndhbWHNpBqTaOvX1tRlpSBDBTI5FxcXDhw4wLRp04iKiiJfvnxMnjyZhg0bAtCvXz8iIyP55ptvCA8Pp3jx4mzevPmVMwoAfP3115w+fZo2bdqgUqn44osv6NWrF1u3btVv061bN/bt20fFihWJjo5m79695M+fP9l+cuXKxZYtWxgyZAhlypQhe/bsdOnShe+//95onwsHBwe2b99O//79qVSpEg4ODrRs2ZIpU6a89XkHDhxg2LBhtGjRgmfPnpErVy4+/vhjXFxcSEhI4MqVKyxdupTHjx/j7e1N7969+frrrwFo2bIlmzZtolatWjx9+pSAgAA6depktNcphBBCpKUHMQ/osasHN57ewMnaiRm1Z1DJq5Kpy3ollfI+E4hmAlFRUbi6uhIZGYmLi0uyx+Lj4wkKCsLX1xc7O7s0OZ5WqyUqKgoXF5d0GeMqjMOcz6Mxvu4zKrVazZYtW2jUqFGK8dQi85DzaB7kPGZMt57e4utdXxMWE4aHvQdz/OdQNPurh9cZ8xy+qV97kVxxFUIIIYTIgs6En6HPnj5EJkSS3yU/8+rOw8fJx9RlvZF5XUoSQgghhBBvtS9kH912dCMyIZLSHqVZ3nB5hm9awcSNq0aj4YcfftDHixYsWJAff/wx2Z3oiqIwcuRIvL29sbe3x9/fn+vXr5uwaiGEEEKIzGvjtY3039ufeE081XNXZ0HdBbjZuZm6rFQxaeM6YcIE5syZw8yZM7l8+TITJkxg4sSJ/Pbbb/ptJk6cyIwZM5g7dy7Hjh3D0dGR+vXrEx8fb8LKhRBCCCEyF0VRmHt2LqOPjkaraGleqDnTa03HwdrB1KWlmknHuB45coSmTZvyySefALpphVavXs3x48cB3Sd42rRpfP/99zRt2hSAZcuWkTNnTv744w99jKkQQgghhHi9F4MFALqV6kbfcn2TJVxmBiZtXD/66CPmz5/PtWvXKFKkCGfPnuXQoUP66YuCgoIICwvD399f/xxXV1c++OADjh49+srGNSEhIdk8mlFRUYDuTriX5xtVq9UoioJWq31rWlNqPR/m8Hy/InMy5/Oo1WpRFAW1Wo2l5btFDWcWz7/nXzXXsMg85DyaBzmPppOgSeDbw9+y9+5eVKgYWnEobYq0ISkpyaD9GPMcpnafJm1chw8fTlRUFMWKFcPS0hKNRsO4ceNo164dAGFhYQApJo/PmTOn/rGXjR8/njFjxqRYv2PHDhwckl8Kt7KywsvLi+joaBITE9PiJek9e/YsTfcnTMMcz2NiYiJxcXEcOHDA4B9amdXOnTtNXYJIA3IezYOcx/QVp41jRcwK7mjuYIklnzl8hvMNZ7bc2PLO+zTGOYyNjU3VdiZtXNetW8fKlStZtWoVJUqU4MyZMwwYMAAfHx86duz4TvscMWIEgwYN0i9HRUWRJ08e6tWr98p5XENCQnByckqz+SwVReHZs2c4Oztnusvv4j/mfB7j4+Oxt7enevXqWWIe1507d1K3bl2ZNzITk/NoHuQ8pr8HsQ/ou7cvdzR3cLJ2Ymr1qVTIWeGd92fMc/j8HfK3MWnjOmTIEIYPH65/y79UqVLcuXOH8ePH07FjR7y8vAB48OAB3t7e+uc9ePCAsmXLvnKftra22NraplhvbW2d4pOs0WhQqVRYWFik2STzz99Wfr5fkTmZ83m0sLBApVK98nvCXGWl12rO5DyaBzmP6ePm05v02NWDsJgwPO09me0/+7XBAoYyxjlM7f5M+hs5NjY2RVNgaWmpbxp8fX3x8vJi9+7d+sejoqI4duwYVapUSddahfF16tSJZs2aZZj9CCGEEJnR6fDTfLn1S8Jiwsjvkp/ljZanWdNqaiZtXBs3bsy4ceP43//+x+3bt/n999+ZMmUKzZs3B3RXuwYMGMBPP/3E5s2bOX/+PF9++SU+Pj7SmABHjx7F0tJSPyvD66xevRpLS0t69+6d4rF9+/ahUqn0VxZdXV0pV64cQ4cO5f79+2/c7+3bt1GpVJw5c+Z9XsY7e93xp0+fzpIlS4x+/Oeftxc/qlWrZvTjvq2mP/74w6Q1CCGEMJ29wXvptqMbUYlRmSpYILVM2rj+9ttvtGrVil69euHn58fgwYP5+uuv+fHHH/XbDB06lL59+9K9e3cqVapEdHQ027ZtM/uxeamxaNEi+vbty4EDBwgNDX3jdkOHDmX16tWvnf/26tWrhIaGEhgYyLBhw9i1axclS5bk/PnzxirfaFxdXXFzc0uXYwUEBHD//n39x+bNm995X3KnrRBCiPex4doGBuwbQIImgRq5a7Cw3sJMEyyQWiZtXJ2dnZk2bRp37twhLi6Omzdv8tNPP2FjY6PfRqVSMXbsWMLCwoiPj2fXrl0UKVLEOAUpCiTGvP+HOtbw57yQFpYa0dHRrF27lp49e/LJJ5+89gpjUFAQR44cYfjw4RQpUoRNmza9cjtPT0+8vLwoUqQIn3/+OYcPH8bDw4OePXumuqaIiAjatWuHh4cH9vb2FC5cmICAAP3j58+fp3bt2tjb2+Pu7k737t2Jjo5+7f62bdtGtWrVcHNzw93dnU8//ZSbN2/qH/f19QWgXLlyqFQqatasCaQcKpCQkEC/fv3w9PTEzs6OatWqERgYqH/8+VXn3bt3U7FiRRwcHKhWrVqqEtrc3Nzw8vLSf2TPnh3QjZEdO3YsuXPnxtbWlrJly7Jt2zb9855fLV67di01atTAzs6OlStXArBw4UL8/Pyws7OjWLFizJ49W/+8xMRE+vTpg7e3N3Z2duTLl4/x48cDunmQAZo3b45KpdIvCyGEMG+KojDn7BzGHB2jDxaYVmsa9lb2pi4tzZn05qwMRx0LP7/f5XQLwO1dnvhtKNg4pnrzdevWUaxYMYoWLUr79u0ZMGAAI0aMSHEHfEBAAJ988gmurq60b9+eRYsW0bZt27fu397enh49ejBw4EDCw8Px9PR863N++OEHLl26xNatW8mRIwc3btwgLi4OgJiYGOrXr0+VKlUIDAwkPDycrl270qdPn9c23TExMQwaNIjSpUsTHR3NyJEjad68OWfOnMHCwoLjx49TuXJldu3aRYkSJZL9wfOioUOHsnHjRpYuXUq+fPmYOHEi9evX58aNG/pGE+C7775j8uTJeHh40KNHD/r06cPRo0ff+rpfZfr06UyePJl58+ZRrlw5Fi9eTJMmTbh48SKFCxfWbzd8+HAmT55MuXLl9M3ryJEjmTlzJuXKleP06dN069YNR0dHOnbsyIwZM9i8eTPr1q0jb968hISEEBISAkBgYCCenp4EBATQoEEDs5+jVQghhC5Y4OdjP7Pu2joAupfuTp+yfcxuRpznpHHNpBYtWkT79u0BaNCgAZGRkezfv19/1RF0V/2WLFmij9D9/PPP+eabbwgKCtJfrXyTYsWKAbqrg6lpXIODgylXrhwVK1YESHbFb9WqVcTHx7Ns2TIcHXUN+syZM2ncuDETJkxIMVcvQMuWLZMtL168GA8PDy5dukTJkiXx8PAAwN3dXT8DxctiYmKYM2cOS5YsoWHDhgAsWLCAnTt3smjRIoYMGaLfdty4cdSoUQPQNbuNGzcmPj4+xfy/L/riiy+SNYgrVqygWbNmTJo0iWHDhulnzJgwYQJ79+5l2rRpzJo1S7/9gAEDaNGihX551KhRTJ48Wb/O19eXS5cuMW/ePDp27EhwcDCFCxemWrVqqFQq8uXLp3/u88/H86vAQgghzFt8UjzDDgxjT8geVKj49oNv+byYeaeKSuP6ImsH3ZXP96DVaol69gwXZ2fDplEyICf46tWrHD9+nN9//x3QBSm0adOGRYsWJWtcd+7cSUxMDI0aNQIgR44c1K1bl8WLFycbR/w6z9OjUvtXW8+ePWnZsiWnTp2iXr16NGvWjI8++giAy5cvU6ZMGX3TClC1alW0Wi1Xr159ZeN6/fp1Ro4cybFjx3j06JF+tong4GBKliyZqppu3ryJWq2matWq+nXW1tZUrlyZy5cvJ9u2dOnS+v8/n34tPDz8jW+5T506NVmym7e3N1FRUYSGhiY75vPXe/bs2WTrnjf5oGuyb968SZcuXejWrZt+fVJSEq6uroBuGETdunUpWrQoDRo04NNPP6VevXpv+zQIIYQwM5EJkfTb049T4aewtrBmQvUJ1M1X19RlGZ00ri9SqQx6u/6VtFqw1uj2Y6T5PxctWkRSUhI+Pv8Na1AUBVtbW2bOnKlvchYtWsSTJ0+wt/9vjItWq+XcuXOMGTPmrY3188YutWMlGzZsyJ07d9iyZQs7d+6kTp069O7dm0mTJhn4CnUaN25Mvnz5WLBgAT4+Pmi1WkqWLJnmKWfPvTiH3PNm/W1xr15eXhQqVCjZutROogwka+Sfj/ddsGABH3zwQbLtnl/VLV++PEFBQWzdupVdu3bRunVr/P392bBhQ6qPKYQQInMLiwmj566e3Hh6A2drZ6bXnk4lr0qmLitdmNfM6llAUlISy5YtY/LkyZw5c0b/cfbsWXx8fFi9ejUAjx8/5s8//2TNmjXJtjt9+jQRERHs2LHjjceJi4tj/vz5VK9eXf8WdGp4eHjQsWNHVqxYwbRp05g/fz4Afn5+nD17lpiYGP22hw8fxsLCgqJFU84t9/jxY65evcr3339PnTp18PPzIyIiItk2z8e0ajSa19ZTsGBBbGxsOHz4sH6dWq0mMDCQ4sWLp/p1GcLFxQUfH59kxwTd633TMXPmzImPjw+3bt2iUKFCyT5eHNrh4uJCmzZtWLBgAWvXrmXjxo08efIE0DXfb/p8CCGEyNxuPr1J+y3tufH0Bp72nixpuCTLNK0gV1wznb///puIiAi6dOmiv7L6XMuWLVm0aBE9evRg+fLluLu707p16xRv9Tdq1IhFixbRoEED/brw8HDi4+N59uwZJ0+eZOLEiTx69Oi1sxC8ysiRI6lQoQIlSpQgISGBv//+Gz8/PwDatWvHqFGj6NixI6NHj+bhw4f07duXDh06vHKYQLZs2XB3d2f+/Pl4e3sTHBzM8OHDk23j6emJvb0927ZtI3fu3NjZ2aX4nDg6OtKzZ0+GDBlC9uzZyZs3LxMnTiQ2NpYuXbqk+rUZasiQIYwaNYqCBQtStmxZAgICOHPmjH7mgNcZM2YM/fr1w9XVlQYNGpCQkMCJEyeIiIhg0KBBTJkyBW9vb8qVK4eFhQXr16/Hy8tLP/1X/vz52b17N1WrVsXW1pZs2bIZ7TUKIYRIX6fDT9Nndx+iEqPwdfVlrv9cs5qjNTXkimsms2jRIvz9/VM0aKBrXE+cOMG5c+dYvHixflqkV223efNmHj16pF9XtGhRfHx8qFChAr/88gv+/v5cuHDBoKuSNjY2jBgxgtKlS1O9enUsLS1Zs2YNAA4ODmzfvp0nT55QqVIlWrVqRZ06dZg5c+Yr92VhYcGaNWs4efIkJUuWZODAgfz666/JtrGysmLGjBnMmzcPHx8fmjZt+sp9/fLLL7Rs2ZIOHTpQvnx5bty4wfbt243a1PXr149BgwbxzTffUKpUKbZt28bmzZuTzSjwKl27dmXhwoUEBARQqlQpatSowZIlS/RXXJ2dnZk4cSIVK1akUqVK3L59my1btuiHfUyePJmdO3eSJ08eypUrZ7TXJ4QQIn3tCd6jDxYo41GGZQ2WGbdpTTLOsLz3pVIUAycQzWSioqJwdXUlMjISFxeXZI/Fx8fr77BPq0ADrVZLVFQULi4uZpdxn5WY83k0xtd9RqVWq9myZQuNGjWSbPRMTM6jeZDz+O42XNvAj//8iFbRUjN3TSbWmGjcOVqj7sPKVvBhLyjXTr/amOfwTf3ai8zrN7IQQgghhJl4OVigReEWTK011bhN68NrsKguPLgAe8dBYqzxjvUOZIyrEEIIIUQGo9FqGHdsHOuvrQfSKVggJBBWtYa4J5C9IHTYBDapn64zPUjjKoQQQgiRgbwcLPDdB9/Rplgb4x702nZY1xGS4sCnPLRbD445jHvMdyCNqxBCCCFEBhGZEEnfPX05HX4aGwsbJlSfgH8+/7c/8X2cXgGb+4GigUJ1ofXS95/X3kikcRVCCCGEyADCYsLosbMHNyNv4mzjzG+1f6NCzgrGO6CiwMHJsOffNM0ybaHJDLDMuDfPSeMqhBBCCGFiNyJu0GNXDx7EPsDTwZO5/nMpnO3NUyi+F60Gtg6DwAW65WqDoM5IXYpoBiaNqxBCCCGECZ16cIo+e/rwLPEZvq6+zPOfh7eTt/EOqI6H37vDpT8BFTT4BT7sYbzjpSFpXIUQQgghTGR38G6GHRhGgiaBMh5lmFl7Jm52bsY7YNxTWNMO7hwCSxtoPg9KtjDe8dKYNK5CCCGEECaw7uo6xh0bl47BAqGwohWEXwQbZ/hiFfhWN97xjEAaVzPUqVMnnj59yh9//GHqUoQQQgjxkufBAnPOzgGgReEW/PDhD1hZGLEte3gNVrSAyBBwygntNoB3aeMdz0gkOSsT6tSpEyqVCpVKhY2NDYUKFWLs2LEkJSUBMH36dJYsWWL0Op7X8OJHtWrVjH7ct9UkDbsQQoiMKkmbxNh/xuqb1q9Lf83oKqON27SGHIfF9XRNa/aC0GVHpmxaQa64ZloNGjQgICCAhIQEtmzZQu/evbG2tmbEiBG4urqmWx0BAQE0aNBAv2xjY/PO+1Kr1ZJfLYQQwmzFJ8Uz9MBQ9obsRYWKbz/4ls+LfW7cg17dBus7ZfhggdSSK64vUBSFWHXse3/EJcUZ/BxFUQyq1dbWFi8vL/Lly0fPnj3x9/dn8+bNgO6KbLNmzfTb1qxZk379+jF06FCyZ8+Ol5cXo0ePTra/p0+f0rVrVzw8PHBxcaF27dqcPXv2rXW4ubnh5eWl/8iePTsAWq2WsWPHkjt3bmxtbSlbtizbtm3TP+/27duoVCrWrl1LjRo1sLOzY+XKlQAsXLgQPz8/7OzsKFasGLNnz9Y/LzExkT59+uDt7Y2dnR358uVj/PjxAOTPnx+A5s2bo1Kp9MtCCCGEqUUmRNJ9Z3f2huzFxsKGKTWnGL9pPbUc1rTVNa2F6kKnvzN10wpyxTWZuKQ4Plj1gUmOfaztMRys3z0P2N7ensePH7/28aVLlzJo0CCOHTvG0aNH6dSpE1WrVqVu3boAfPbZZ9jb27N161ZcXV2ZN28ederU4dq1a/pm1BDTp09n8uTJzJs3j3LlyrF48WKaNGnCxYsXKVz4v3nphg8fzuTJkylXrpy+eR05ciQzZ86kXLlynD59mm7duuHo6EjHjh2ZMWMGmzdvZt26deTNm5eQkBBCQkIACAwMxNPTU38V2NLS0uC6hRBCiLRmmmCBSbDnJ91y2XbQeHqGDhZILWlcMzlFUdi9ezfbt2+nb9++r92udOnSjBo1CoDChQszc+ZMdu/eTd26dTl06BDHjx8nPDwcW1tbACZNmsQff/zBhg0b6N69+2v3+8UXXyRrEFesWEGzZs2YNGkSw4YN4/PPdX9NTpgwgb179zJt2jRmzZql337AgAG0aPHfNByjRo1i8uTJ+nW+vr5cunSJefPm0bFjR4KDgylcuDDVqlVDpVKRL18+/XM9PDyA/64CCyGEEOklQZPA0dCjxCXFJVufqElkxukZhMeGS7BAGpDG9QX2VvYca3vsvfah1Wp59uwZzs7OWFikfiSGodNf/P333zg5OaFWq9FqtbRt2zbF2/8vKl06+SBsb29vwsPDATh79izR0dG4u7sn2yYuLo6bN2++sY6pU6fi7/9fhrK3tzdRUVGEhoZStWrVZNtWrVo1xfCDihUr6v8fExPDzZs36dKlC926ddOvT0pK0o/b7dSpE3Xr1qVo0aI0aNCATz/9lHr16r2xRiGEEMKYIuIj6L27N+cfnX/tNgVcCzDXf67xgwU2dYPLmwEVNJwAH3xtvOOZgDSuL1CpVO/1dj3oGtckqyQcrB0MalwNVatWLebMmYONjQ0+Pj5YWb35VL5805NKpUKr1QIQHR2Nt7c3+/btS/E8Nze3N+7Xy8uLQoUKJVsXFRX19hfwL0dHR/3/o6OjAViwYAEffJB8yMbzq7rly5cnKCiIrVu3smvXLlq3bo2/vz8bNmxI9TGFEEKItHL32V167urJ7ajbONs4Uzx78RTb5HbOzcAKA3G1NeLN03FPdeNZ7xzOlMECqSWNaybl6OiYomF8V+XLlycsLAwrK6s0uaHJxcUFHx8fDh8+TI0aNfTrDx8+TOXKlV/7vJw5c+Lj48OtW7do167dG/ffpk0b2rRpQ6tWrWjQoAFPnjwhe/bsWFtbo9Fo3vs1CCGEEG9z9clVeuzqwaO4R3g7ejO37lwKuBZI/0JeDBawdYHPV2a6YIHUksZV4O/vT5UqVWjWrBkTJ06kSJEihIaG8r///Y/mzZsnezs/tYYMGcKoUaMoWLAgZcuWJSAggDNnzuhnDnidMWPG0K9fP1xdXWnQoAEJCQmcOHGCiIgIBg0axJQpU/D29qZcuXJYWFiwfv16vLy89FeG8+fPz+7du6latSq2trZky5btXT4lQgghxBsdv3+c/nv7E62OpnC2wsz1n4ung2f6F/LwKqxo+W+wgBe03wBepdK/jnQijatApVKxZcsWvvvuOzp37szDhw/x8vKievXq5MyZ85322a9fPyIjI/nmm28IDw+nePHibN68OdmMAq/StWtXHBwc+PXXXxkyZAiOjo6UKlWKAQMGAODs7MzEiRO5fv06lpaWVKpUiS1btuiHZUyePJlBgwaxYMECcuXKxe3bt9+pfiGEEOJ1tt3exrcHv0WtVVMxZ0Wm156Oi41L+hcSchxWtYa4CHAvBO03QbZ8b39eJqZSDJ1ANJOJiorC1dWVyMhIXFySf1HFx8cTFBSEr68vdnZ2aXI8rVZLVFQULi4uRh3jKozLnM+jMb7uMyq1Ws2WLVto1KiRhFtkYnIezYO5nMeVl1cy4fgEFBTq5qvL+I/HY2tpm/6FvBgskKsitF0Hju5vfdr7MOY5fFO/9iK54iqEEEII8RaKojD91HQWXVgEwOdFP2d45eFYWphgzvBTy+CvAaBooHA9+GwJ2Di+7VlmQRpXIYQQQog3UGvVjD4yms03dQmV/cr1o2uprqjSe25URYEDk2Cv+QULpJY0rkIIIYQQrxGrjuWb/d9w6N4hLFWWjKoyiuaFm6d/IVoNbB0KgQt1yx8Phtrfm02wQGpJ4yqEEEII8QpP4p/QZ3cfzj86j52lHZNrTqZ6bhNMM6WOh01d4fJf6IIFJsIHr0+1NGfSuKIbtyJEViFf70II8XYvBgu42royq84syniUSf9CXg4WaLEASjRL/zoyiCzduD6/Iy42NhZ7e8MiV4XIrGJjY4GUaWpCCCF0rjy5Qs9dPTNIsEBLCL/0b7DAKvD9OP3ryECydONqaWmJm5sb4eHhADg4OLz3QGutVktiYiLx8fFmN41SVmKO51FRFGJjYwkPD8fNzU0foyuEEOI/x+4fo//e/sSoY0wfLLC8BUTd/TdYYCN4lUz/OjKYLN24Anh5eQHom9f3pSgKcXFx2Nvbp//dhiLNmPN5dHNz03/dCyGE+M+2oG2MODSCJG2SaYMFgo/pggXin4J7YeiwCdzypn8dGVCWb1xVKhXe3t54enqiVqvfe39qtZoDBw5QvXp1eSs2EzPX82htbS1XWoUQ4hUyTrDAVljfOV2DBTKTLN+4PmdpaZkmv9AtLS1JSkrCzs7OrBqerEbOoxBCmJ/D9w4z+uhoHsU9SvFYkjYJMHGwwMml8PcAULRZLlggtaRxFUIIIYTZ++vmX4w8PJIkJemVj1uprOhVtpcJgwV+hb3jdMtZMFggtaRxFUIIIYTZUhSFJReXMOXkFAAa+jZkUIVBqEjenDpYO+Bs45z+BWo1sGUwnFisW86iwQKpJY2rEEIIIcySVtEy6cQkll9aDsCXxb/km4rfYKHKILPFqONhYxe48jdZPVggtaRxFUIIIYTZSdQk8v2h79l6eysAgysOpmOJjiau6gVxEbC6LQQfkWABA0jjKoQQQgizEp0YzYB9Azh2/xhWKit+rPYjnxb41NRl/SfyHqxsJcEC70AaVyGEEEKYjUdxj+i5qydXnlzB3sqeaTWn8VGuj0xd1n/Cr+jSsCRY4J1I4yqEEEIIs3An6g5f7/yae9H3yG6Xndn+synhXsLUZf0n+B9Y1UaCBd6DNK5CCCGEyPQuPLpAr129iEiIII9zHub6zyWvSwZqCq/8DzZ8BUnxEizwHqRxFUIIIUSmdujeIQbtG0RcUhx+2f2Y7T+bHPY5TF3Wf04ugb8H/hssUB8+C5BggXeUQeaDEEIIIYQw3F83/6Lv7r7EJcVRxbsKAQ0CMk7TqiiwbwL81V/XtJZrr7sRS5rWdyZXXIUQQgiR6SiKQsDFAKaenApAI99G/FT1J6wzStqUVgP/+wZOBuiWJVggTUjjKoQQQohMRato+TXwV1ZcXgFAx+IdGVRxUAYKFoiDjV3/CxZo9CtU7mbqqsyCNK5CCCGEyDQSNYl8d+g7tt3eBmTUYIEvIPioLlig5UIo3tTUVZkNaVyFEEIIkSlEJ0YzYO8AjoUdw8rCip+q/sQnBT4xdVn/ibynm6P14WWwdYUvVkH+aqauyqxI4yqEEEKIDEOj1bD6ymquRVxL8di5h+e4GXkTBysHptaaykc+GTRYwNlbFyyQMwPNIWsmpHEVQgghRIaQoElg+IHh7Are9dptsttlZ47/HIq7F0/Hyt7ixWCBHEV0TasECxiFNK5CCCGEMLmoxCj67enHyQcnsbawpmOJjjhaJ582ytrCmvr56+Pl6GWiKl/hxWCB3JV0wQIO2U1dldmSxlUIIYQQJvUg5gE9dvXgxtMbOFk7MaP2DCp5VTJ1WW/3YrBAkQbQKgBsHExdlVmTxlUIIYQQJnPr6S2+3vU1YTFheNh7MMd/DkWzFzV1WW+mKLB/Iuz7WbdcrgN8Og0spa0yNvkMCyGEEMIkzoSfoc+ePkQmRJLfJT9z684ll1MuU5f1Zi8HC1QfArW+k2CBdCKNqxBCCCHS3b6QfQzZP4R4TTylc5RmZp2ZZLPLZuqy3kyCBUxOGlchhBBCpKuN1zYy9p+xaBUt1XNX59fqv+JgncHHhsY+0QULhPwDlrb/Bgs0MXVVWY40rkIIIYRIF4qisPDCQmafmw1A80LNGVllJFYWGbwdibz7b7DAlX+DBVZD/qqmripLyuBfKUIIIYQwBxqthr/i/uL4ueMAdCvVjb7l+qLK6GNDwy//GyxwT4IFMgBpXIUQQgiRZs6EnyH4WXCK9btu7+J44nFUqBheeTht/dqaoDoDBf8Dq1pDfKQEC2QQ0rgKIYQQ4r0pisLMMzOZf27+a7exxJLx1cbTsGDDdKzsHV3+GzZ2+TdYoDK0XSvBAhmANK5CCCGEeC9J2iTGHh3L7zd+B6CSVyVsLG2SbWNrYUuBiAL45/U3RYmGOREA/xskwQIZkDSuQgghhHhncUlxDNk/hP1392OhsuCHD3+gVZFWKbZTq9Vs2bLFBBUaQFFg/wTYN163LMECGY6cCSGEEEK8k6fxT+m9pzfnHp7D1tKWidUnUjtvbVOX9W5SBAsMhVrfSrBABiONqxBCCCEMFhodytc7v+Z21G1cbFyYWWcm5TzLmbqsd/NysMAnk6BSV1NXJV5BGlchhBBCGORaxDV67uxJeFw4OR1yMq/uPAq6FTR1We9GggUyFWlchRBCCJFqgWGB9N/Tn2fqZxRyK8Qc/zl4OXqZuqx382KwgJ0rfLEG8n1k6qrEG0jjKoQQQohU2XF7B8MPDketVVPeszwzas/A1dbV1GW9m2TBAj7/BgsUN3VV4i2kcRVCCCHEW62+sprxx8ajoFAnbx1++fgX7KzsTF3Wu7lzFFa3+TdYoOi/wQJ5TF2VSAVpXIUQQgjxWoqi8Nvp31hwfgEArYu05tsPvsXSwtLElb0jCRbI1KRxFUIIIcQrvRws0Ltsb74u/TWqzDpF1InFuimvFC0UbQQtF0mwQCYjjasQQgghUohLimPw/sEcuHsAC5UFIz8cScsiLU1d1rtRFNj3C+z/Rbdc/kv4ZKoEC2RCcsaEEEIIkUxEfAR99vTRBwv8Wv1XauWtZeqy3o0mSRffemqpbrnGMKg5QoIFMilpXIUQQgih93KwwKw6syjrWdbUZb0bdRxs+AqubkGCBcyDNK5CCCGEAODqk6v03NWTh3EP8XL0Yp7/PAq4FTB1We8m9gms/hxCjkmwgBmRxlUIIYQQBIYF0m9PP6LV0RRyK8Rc/7nkdMxp6rLeTeRdWN4CHl2VYAEzI42rEEIIkcVtv72dEQdHmEewwINLumCBZ6ESLGCGpHEVQgghsrBVl1fxy/FfUFDwz+vPL9V/wdbS1tRlvZs7R3TDA54HC3TYBK65TV2VSEPSuAohhBBZkKIozDg9g4XnFwLQpmgbRlQekYmDBf6CDV1AkwB5PtAND5BgAbMjjasQQgiRxai1asYcGcOfN/8EoE/ZPnQv3T3zBgsELoItg/8LFmi1GKztTV2VMAJpXIUQQogsJFYdy+D9gzl476CZBAuMh/0TdMvlO8InUyRYwIzJmRVCCCGyiIj4CPrs7sO5R7pggUk1JlEzT01Tl/VuNEnwv4FwapluWYIFsgRpXIUQQogs4F70PXrs7MHtqNu42roys/bMzBsskBirCxa4thVUFtBoElTqYuqqRDqQxlUIIYQwc1efXKXHrh48intkHsECq9rA3eO6YIFWi8CvsamrEulEGlchhBDCjB2/f5z+e/ubR7DA0xDdHK36YIG1kK+KqasS6UgaVyGEECKTC4kKYeqpqTyOe5zisfOPzqPWqqmQswIzas/AxcbFBBWmgQcX/w0WuC/BAlmYNK5CCCFEJnbx8UV67erFk/gnr90m0wcL3D4Mq7+AhEjwKKZrWiVYIEuSxlUIIYTIpI6EHmHg3oHEJsXil92PrqW6ppiL1dXGlQo5K2TeYIFLm2Fj13+DBT6EL1ZLsEAWZmHqAu7du0f79u1xd3fH3t6eUqVKceLECf3jiqIwcuRIvL29sbe3x9/fn+vXr5uwYiGEEML0/r71N7139SY2KZYPvD9gcf3F1Mtfj7r56ib7qOxdOfM2rYELYd2Xuqa16Cfw5R/StGZxJm1cIyIiqFq1KtbW1mzdupVLly4xefJksmXLpt9m4sSJzJgxg7lz53Ls2DEcHR2pX78+8fHxJqxcCCGEMJ2lF5cy4uAIkpQkGvo2ZE6dOTjZOJm6rLSjKLD3Z/jfN4ACFTpB62WShiVMO1RgwoQJ5MmTh4CAAP06X19f/f8VRWHatGl8//33NG3aFIBly5aRM2dO/vjjDz7//PMU+0xISCAhIUG/HBUVBYBarUatVhvrpeg9P0Z6HEsYj5xH8yDn0TzIefyPVtEy7fQ0VlxZAUDbom0ZVH4QaHUxrhlZqs+jNgnLrYOxOKN7jZqPh6L9eAhoFcjgr9HcGfN7MbX7VCmKoqT50VOpePHi1K9fn7t377J//35y5cpFr1696NatGwC3bt2iYMGCnD59mrJly+qfV6NGDcqWLcv06dNT7HP06NGMGTMmxfpVq1bh4OBgtNcihBBCGFOSksSm2E2cU58DoIFdA6raVk0xpjUzs9QmUOH2bLwjT6Og4myejtzJUdvUZYl0EBsbS9u2bYmMjMTF5fUzX5i0cbWzswNg0KBBfPbZZwQGBtK/f3/mzp1Lx44dOXLkCFWrViU0NBRvb2/981q3bo1KpWLt2rUp9vmqK6558uTh0aNHb/xEpBW1Ws3OnTupW7cu1tbWRj+eMA45j+ZBzqN5kPMIMeoYhhwcwj9h/2ClsmLkhyP51PdTU5dlkLeex7gILNe1w+LucRQrOzTN5qMUbZT+hYrXMub3YlRUFDly5Hhr42rSoQJarZaKFSvy888/A1CuXDkuXLigb1zfha2tLba2Kaf7sLa2TtcfeOl9PGEcch7Ng5xH85BVz+OjuEf02t2Ly08uY29lz5SaU6iWq5qpy3pnrzyPLwULqL5Yi5UEC2RYxvheTO3+THpzlre3N8WLJ5882M/Pj+DgYAC8vLwAePDgQbJtHjx4oH9MCCGEMFfBUcF02NKBy08uk90uO4vrL87UTesrPbgEi+rpmlaXXPDVdknDEq9l0sa1atWqXL16Ndm6a9eukS9fPkB3o5aXlxe7d+/WPx4VFcWxY8eoUkW+qIUQQpivi48u0mFrB+5G3yWXUy6WNVxGyRwlTV1W2rp9GBY3gGehumCBLjvA08/UVYkMzKRDBQYOHMhHH33Ezz//TOvWrTl+/Djz589n/vz5AKhUKgYMGMBPP/1E4cKF8fX15YcffsDHx4dmzZqZsnQhhBDCaI7cO8KAfQOIS4rDL7sfs/1nk8M+h6nLSlsvBgvkraILFrDP9vbniSzNpI1rpUqV+P333xkxYgRjx47F19eXadOm0a5dO/02Q4cOJSYmhu7du/P06VOqVavGtm3b9Dd2CSGEEObkr5t/MfLwSJKUJD7w/oBpNaeZ1xytoAsW+N9gQIFin0LLhTJHq0gVk0e+fvrpp3z66evvjFSpVIwdO5axY8emY1VCCCFE+lIUhaUXlzL55GQAGvo2ZFzVcVhbmtENaYqCxb7xcFj3GqnQGT6ZDJk12UukO5M3rkIIIURWp1W0TDoxieWXlgPQ3q89QyoNwUJl8mT2tKNNomzIYiwf79ct1/wWagwFM5qHVhifNK5CCCFEOjkTfoYbT2+kWH8k9Ag77+wEYFCFQXQq0cmsggVIjMVyfUfyPd6PorJA9elUXYyrEAaSxlUIIYQwMkVRWHh+ITNOz3jtNlYqK8ZWHUvjgo3TsbJ0EPsEVrXG4m4gGpU1SstFWJVsauqqRCYljasQQghhRBqthl+O/8Kaq2sA+MDrA+xfuhHJxsKGz4t9TiWvSqYo0XieBsPyFvD4OoqdG0fy9OFDScMS70EaVyGEEMJIEjQJjDg4gp13dqJCxbDKw2jn1+7tTzQHYRd0aVjRYeCSi6TP1/Ek8KapqxKZnDSuQgghhBFEJUbRf09/Tjw4gbWFNT9//DMN8jcwdVnp4/YhWN0WEiLBww/abwQHT0AaV/F+pHEVQggh3pFW0XL+0Xli1DHJ1mu0Gqaemsr1iOs4WjsyvdZ0PvD+wERVprOLf8CmbqBJhLwfwRerdMECarWpKxNmIFWN67lz51K9w9KlS79zMUIIIURmEZ8Uz5ADQ9gXsu+12+Swz8Ec/zkUy14s3eoyqeMLYMsQJFhAGEuqGteyZcuiUqlQFOWt03NoNJo0KUwIIYTIqCITIumzuw9nHp7BxsIGX1ffFNt4OXoxvPJwcjvnNkGF6UxRYM+PcPDfYIGKX0GjSRIsINJcqhrXoKAg/f9Pnz7N4MGDGTJkCFWqVAHg6NGjTJ48mYkTJxqnSiGEECKDCIsJo8fOHtyMvImzjTMza8+kfM7ypi7LdDRJ8Fd/OLNCt1zrO6g+RIIFhFGkqnHNly+f/v+fffYZM2bMoFGj/6azKF26NHny5OGHH36gWbNmaV6kEEIIkRFcj7hOj109CI8Nx9PBk7n+cymcrbCpyzKdxBhY3xmubweVBXw6DSp0NHVVwowZfHPW+fPn8fVN+ZaIr68vly5dSpOihBBCiIzm5IOT9N3Tl2eJzyjoWpC5defi5ehl6rJMJ+YxrGoN906AlR20CoBiMkerMC6DQ5D9/PwYP348iYmJ+nWJiYmMHz8ePz+/NC1OCCGEyAh239lN9x3deZb4jHKe5VjacGnWbloj7sDi+rqm1c4NvtwsTatIFwZfcZ07dy6NGzcmd+7c+hkEzp07h0ql4q+//krzAoUQQghTWnd1HeOOjUOraKmVpxYTq0/EzsrO1GWZTrJggdy6OVo9s8isCcLkDG5cK1euzK1bt1i5ciVXrlwBoE2bNrRt2xZHR8c0L1AIIYQwBUVRmH12NnPPzgWgVZFWfPfBd1hZZOEp0IMOwpq2kBAFnsWh3QZwzWXqqkQW8k7ffY6OjnTv3j2taxFCCCEyhCRtEj/98xMbr28EoFeZXvQo0+OtU0KatdcFCwiRjgwe4wqwfPlyqlWrho+PD3fu3AFg6tSp/Pnnn2lanBBCCJHe4pLiGLhvIBuvb8RCZcEPH/5Az7I9s3bTemw+rO+ka1qLfQodNknTKkzC4MZ1zpw5DBo0iIYNGxIREaEPHMiWLRvTpk1L6/qEEEKIdBOZEEn3Hd3ZF7IPW0tbptScQuuirU1dlukoCuweC1v/TcOq2AVaL5M0LGEyBjeuv/32GwsWLOC7777Dyuq/kQYVK1bk/PnzaVqcEEIIkV7uR9/ny61fcubhGZxtnJlfdz518tYxdVmmo1HDn33+S8Oq9T18MlnSsIRJGTzGNSgoiHLlyqVYb2trS0xMTJoUJYQQQqSnaxHX6LmzJ+Fx4eR0yMlc/7kUylbI1GWZTmKMbmjA9R0SLCAyFIOvuPr6+nLmzJkU67dt2ybzuAohhMh0ToSdoNPWToTHhVPQtSArGq3I2k1rzGNY2kTXtFrZw+erpGkVGYbBV1wHDRpE7969iY+PR1EUjh8/zurVqxk/fjwLFy40Ro1CCCGEUey8s5PhB4aTqE2knGc5fqv9G662rqYuy3Qi7ujmaH18XXfzVdt1kKeyqasSQs/gxrVr167Y29vz/fffExsbS9u2bfHx8WH69Ol8/vnnxqhRCCGEeGdXnlxh9JHRhMWEpXjsSfwTFBQJFgAIOw8rWumCBVzz6IIFPIqauiohknmneVzbtWtHu3btiI2NJTo6Gk9Pz7SuSwghhHhv/9z/hwF7BxCjfv09GK2LtGbEByOyeLDAAVjT7r9ggfYbwcXH1FUJkYLB36W1a9dm06ZNuLm54eDggIODAwBRUVE0a9aMPXv2pHmRQgghhKG2Bm3l20PfkqRNopJXJYZUHIKFKvmtHU42TuRyyuLJTxd/h03ddXO05quqG9Nq72bqqoR4JYMb13379pGYmJhifXx8PAcPHkyTooQQQoj3sfzSciYGTgSgXr56jP94PDaWNiauKgM6Ng+2DgMU8GsMLRaCdRYeLiEyvFQ3rufOndP//9KlS4SF/TdWSKPRsG3bNnLlyuJ/tQohhDApraJl2qlpBFwIAKBtsbYMqzwsxZXWLO95sMChKbrlSl2h4USZo1VkeKluXMuWLYtKpUKlUlG7du0Uj9vb2/Pbb7+laXFCCCFEaqm1akYdHsVft/4CoH/5/nQp2SVrR7W+ikYNm/vB2VW65drfw8eDQT5PIhNIdeMaFBSEoigUKFCA48eP4+HhoX/MxsYGT09PLC3lLzUhhBDpL1Ydy6B9gzgcehhLlSWjPxpNs0LNTF1WxpMYA+s6wo2doLKExtOg/JemrkqIVEt145ovXz4AtFqt0YoRQgghDPUk/gm9d/XmwuML2FvZM6nGJKrnrm7qsjKemMew6jO4d1IXLPDZEijawNRVCWEQgwf9jB8/nsWLF6dYv3jxYiZMmJAmRQkhhBCpEfIshA5bOnDh8QXcbN1YWG+hNK2vEnEbFtfTNa322aDjZmlaRaZkcOM6b948ihUrlmJ9iRIlmDt3bpoUJYQQQrzN5ceX6bClA8HPgsnllIvlDZdT2qO0qcvKeMLOw6J68PiGLljgq+2ShiUyLYOnwwoLC8Pb2zvFeg8PD+7fv58mRQkhhBBvcjT0KAP3DSRGHUPRbEWZ4z8HDwePtz8xq0kWLFAC2m+QYAGRqRl8xTVPnjwcPnw4xfrDhw/j4yPfDEIIIYxra9BWeu3uRYw6hspelQloECBN66tc2AQrWuqa1nxVofMWaVpFpmfwFddu3boxYMAA1Gq1flqs3bt3M3ToUL755ps0L1AIIYR47sVggfr56/NztZ8lWOBVkgULNIEWCyRYQJgFgxvXIUOG8PjxY3r16qVP0LKzs2PYsGGMGDEizQsUQgghtIqWaaensezyMgDa+bVjaKWhEizwMkWB3WPg0FTdsgQLCDNjcOOqUqmYMGECP/zwA5cvX8be3p7ChQtja2trjPqEEEJkcWqtmk2xmzhz+QwgwQKvJcECIgswuHF9LiwsjCdPnlC9enVsbW1RFEV+iAghhEhTsepYBuwfwBn1GSxVloz5aAxNCzU1dVmmdelPuHsi5frQ03D74L/BAtOhfIf0r00IIzO4cX38+DGtW7dm7969qFQqrl+/ToECBejSpQvZsmVj8uTJxqhTCCFEFvM47jG9d/fm4uOLWGPNlBpTqJmvpqnLMh2tFnZ8D//Mev02VvbQeikUqZ9+dQmRjgxuXAcOHIi1tTXBwcH4+fnp17dp04ZBgwZJ4yqEEOK9hTwLocfOHgQ/C8bN1o3PrT+nqk9VU5dlOkmJ8EdPuLBBt1y2PThkS76NhRWUbAVeJdO/PiHSicGN644dO9i+fTu5c+dOtr5w4cLcuXMnzQoTQgiRNV16fIleu3rxOP4xuZxyMbPmTC4eumjqskwn4RmsbQ+39uma06azoUwbU1clhEkYfDtmTEwMDg4OKdY/efJEbtASQgjxXo6GHqXzts48jn9M0WxFWd5wOflc8pm6LNOJDocln+iaVmtHaLtOmlaRpRncuH788ccsW7ZMv6xSqdBqtUycOJFatWqlaXFCCCGyji23ttBrdy9ik2IlWADg8U1YVBfunwWHHNDpbyhUx9RVCWFSBg8VmDhxInXq1OHEiRMkJiYydOhQLl68yJMnT16ZqCWEEEK8zbKLy/j1xK+ABAsAcO8UrPwMYh9BtvzQfhO4FzR1VUKYnMGNa8mSJbl27RozZ87E2dmZ6OhoWrRoQe/evfH29jZGjUIIIcyUVtEy9eRUllxcAkiwAAA3dsPaDqCOAe8y0G4DOHmauiohMgSDG9fg4GDy5MnDd99998rH8ubNmyaFCSGEMG9qjZqRR0by962/ARhQfgBflfwqa88JfnYt/NkLtElQoCa0WQG2zqauSogMw+A/aX19fXn48GGK9Y8fP8bX1zdNihJCCGHeYtQx9NnTh79v/Y2lypKfqv5El1JZOA1LUeDwDPi9u65pLfUZtF0vTasQLzH4iuvrErKio6Oxs7NLk6KEEEJkflpFy9mHZ4lKiEqxfu65uVx6fAl7K3sm15jMx7k/NlGVGcDLwQJV+kDdH8EiCw+XEOI1Ut24Dho0CNDNIvDDDz8kmxJLo9Fw7NgxypYtm+YFCiGEyHwSNYl8e+hbtt/e/tptstlmY1adWZTyKJWOlWUwSQnwR6//ggXq/QQf9TVtTUJkYKluXE+fPg3orrieP38eG5v/7va0sbGhTJkyDB48OO0rFEIIkak8S3zGgL0DOB52HCsLK4plK5Zimxz2Ofim4jfkd82f/gVmFPFRumCBoP26YIFmc6B0a1NXJUSGlurGde/evQB07tyZ6dOn4+LiYrSihBBCZE4PYx/Sa3cvrjy5goOVA9NqTaOKTxVTl5XxPHsAK1tB2DmwcYLWy2SOViFSweAxrgEBAQDcuHGDmzdvUr16dezt7V879lUIIUTWcDvyNj129eBe9D3c7dyZ7T+b4u7FTV1WxvP4JixvDk/v6IIF2m8An3KmrkqITMHgkd9PnjyhTp06FClShEaNGnH//n0AunTpwjfffJPmBQohhMj4zj08x5dbv+Re9D3yOudleaPl0rS+yr2TsKiermnN5gtddkjTKoQBDG5cBwwYgLW1NcHBwclu0GrTpg3btm1L0+KEEEJkfAfvHqTrjq5EJERQwr0EyxouI49zHlOXlfHc2AVLGuvSsLzL6JpWScMSwiAGDxXYsWMH27dvJ3fu3MnWFy5cmDt37qRZYUIIITK+P2/8yagjo9AoGqr6VGVKzSk4WDu8/YlZzdk18Gfvf4MFakGb5TJHqxDvwODGNSYmJtmV1ueePHmCra1tmhQlhBAiY1MUhUUXFjH91HQAGhdozJiqY7C2sDZxZRmMosCRGbBzpG651GfQdDZY2bz5eUKIVzJ4qMDHH3/MsmXL9MsqlQqtVsvEiROpVatWmhYnhBAi49EqWiYETtA3rZ1LdmZctXHStL5Mq4Xt3/3XtFbpA83nS9MqxHsw+IrrxIkTqVOnDidOnCAxMZGhQ4dy8eJFnjx5wuHDh41RoxBCiAzi5WCBoZWG0qF4BxNXlQElJcAfPeHCRt2yBAsIkSYMblxLlizJtWvXmDlzJs7OzkRHR9OiRQt69+6Nt7e3MWoUQgiRATxLfEb/vf0JDAvEysKKn6v9TEPfhqYuK+NJFixg/W+wwGemrkoIs2Bw4wrg6urKd999l9a1CCGEyKAexj6k566eXI24iqO1I9NrTecD7w9MXVbG83KwQJvlULC2qasSwmykqnE9d+5cqndYunTpdy5GCCFExhMUGUSPnT0IjQnF3c6dOf5z8HP3M3VZGc+LwQKOHtBuvczRKkQaS1XjWrZsWVQqVYp0LEVRAJKt02g0aVyiEEIIUzn38By9d/fmacJT8rnkY67/XHI75377E7Oaeydh5WcQ+1gXLNBhE2QvYOqqhDA7qZpVICgoiFu3bhEUFMTGjRvx9fVl9uzZnDlzhjNnzjB79mwKFizIxo0bjV2vEEKIdHLg7gG67ujK04SnlHQvybKGy6RpfZXru2DJp7qm1busLlhAmlYhjCJVV1zz5cun//9nn33GjBkzaNSokX5d6dKlyZMnDz/88APNmjVL8yKFEEKkrz9u/MHoI6N1wQK5qjKlhgQLvNKLwQIFa0Pr5WDrZOqqhDBbBt+cdf78eXx9fVOs9/X15dKlS2lSlBBCCNN4OVigScEmjP5otMzR+jJFgcPTYdco3XKp1tB0lszRKoSRGRxA4Ofnx/jx40lMTNSvS0xMZPz48fj5yWB9IYTIrDRaDb8c/0XftH5V8it+qvqTNK0v02ph24j/mtaP+kLzedK0CpEODL7iOnfuXBo3bkzu3Ln1MwicO3cOlUrFX3/9leYFCiGEML4ETQLfHvyWHXd2oELF0EpDaV+8vanLyniSEuD3HnBxk2653jj4qI9paxIiCzG4ca1cuTK3bt1i5cqVXLlyBYA2bdrQtm1bHB0d07xAIYQQxvVysMD4auNp4NvA1GVlPPFRsLYdBB34N1hgNpRubeqqhDCKqHg1zrZWyWaOygjeKYDA0dGR7t27p3UtQggh0ll4bDg9d/XkWsQ1CRZ4k2dh/wYLnJdgAWH2bj6M5stFx+lQJR89ahQ0dTnJGDzGVQghhHm4FXmL9lvacy3iGjnsc7CkwRJpWl/l0Q1YVFfXtDp6QKf/SdMqzNbp4AhazTnCvadxrAsMIS4xY83P/05XXIUQQmRuZx+epc/uPhIs8DZ3T8IqCRYQWcOeKw/otfIU8WotZXK7srhTJextLE1dVjLSuAohRBazP2Q/g/cPJl4TT0n3kszyn0V2u+ymLivjub4T1n0J6lhdsEC7DeDkYeqqhDCKdSdCGLHpPBqtQs2iHsxqWx5H24zXJma8ioQQQhjN79d/Z8zRMRIs8DZnVsPmPhIsIMyeoijM3neTX7dfBaBl+dz80rIU1pYZczSpwY1rSEgIKpWK3Ll1bykdP36cVatWUbx4cblhSwghMihFUVhwfgG/nf4NkGABQDcM4NgcUMclX58UDzd26f5fug00mSlztIpMb/+1h6wLDEGt0SZbHxWv5p9bTwDoWbMgQ+sXzXAzCbzI4Ma1bdu2dO/enQ4dOhAWFkbdunUpUaIEK1euJCwsjJEjRxqjTiGEEO/oebDAmqtrAOhSsgv9y/fP0L+cjO7KFtjQWdekvk7V/lBnNFhkzCtPQqTWymN3+OGPC2iVVz+uUsHIT4vTuWrKZNSMxuDG9cKFC1SuXBmAdevWUbJkSQ4fPsyOHTvo0aOHNK5CCJGBJGgSGHFwBDvv7ESFimGVh9HOr52pyzKtk0vh7wGgaKGQPxT7NOU27gXBt3q6lyZEWlIUhWm7rjN993UAmpX1obKve4rtSvi4UCaPWzpX924MblzVajW2trYA7Nq1iyZNmgBQrFgx7t+/n7bVCSGEeGdRiVH039OfEw9OYG1hzc/Vfs7awQKKAgd+hb3jdMvl2sOn08FSbvcQ5idJo+WHPy+y+ngwAP1qF2Jg3SKZ/p0Wg9//KFGiBHPnzuXgwYPs3LmTBg10PwRDQ0Nxd0/ZxQshhEh/4bHhdNrWiRMPTuBo7cgc/zlZu2nVauB/g/5rWj8erBu7Kk2rMEPxag09V55i9fFgVCr4sVlJBtXL2GNXU8vg79gJEybQvHlzfv31Vzp27EiZMmUA2Lx5s34IgRBCCNO5FXmLHjt7cD/mPjnsczDHfw7FshczdVmmo46DjV3hyt+AChr9CpW7mboqId6LoiisP3mXy/ejUjx26k4EZ+9GYmNlwYzPy9KgpLcJKjQOgxvXmjVr8ujRI6KiosiWLZt+fffu3XFwkClVhBDClM4+PEvv3b2JTIiUYAGAuAhY/QUEHwVLG2i5EIo3NXVVQrwXtUbL8I3n2Xjq7mu3cbGzYmHHSlT2Na85mt/pPRJFUTh58iQ3b96kbdu2ODs7Y2NjI42rEEKYkAQLvCTyHqxoCQ8vg60rfLEK8lczdVVCvJfYxCR6rTzFvqsPsbRQ0f6DvDjZJW/nrC0taFo2F745HE1UpfEY3LjeuXOHBg0aEBwcTEJCAnXr1sXZ2ZkJEyaQkJDA3LlzjVGnEEKIN3gxWKBarmpMrjE5awcLhF+BFS0g6h44e0P7jZCzhKmrEuK9PIlJpPOSQM6GPMXO2oJZbctTxy+nqctKVwbfnNW/f38qVqxIREQE9vb2+vXNmzdn9+7daVqcEEKIN1MUhfnn5jPyyEg0ioYmBZswo/aMrN20Bv8Di+vrmtYcRaDLDmlaRaYX8iSWVnOOcDbkKW4O1qzq9mGWa1rhHa64Hjx4kCNHjmBjkzxFJH/+/Ny7dy/NChNCCPFmLwcLdC3VlX7l+pnFncNv9SwMLv8FmsTk6xOi4dAUXbBA7krQdh04ZOHhEiJTiUlI4n/n7hMVr062XqsoLDgYxMNnCeRys2fpV5Up5Jk1I4gNbly1Wi0ajSbF+rt37+Ls7JwmRQkhhHizLB0sEHZBN3Y1Ouz12xRpAK0CwCYLX3kWmcrDZwl0XnKcC/dSzhLwXDEvZ5Z+VZmcLnbpWFnGYnDjWq9ePaZNm8b8+fMBUKlUREdHM2rUKBo1apTmBQohhEguRbDAxz/TIH8WmaM16CCsaQsJUeBeCHzKp9wmZwmo0kfmaBWZxu1HMXy5+DjBT2Jxd7Th48I5UmyT09WOXjUL4WpvbYIKMw6Dv6snT55M/fr1KV68OPHx8bRt25br16+TI0cOVq9ebYwahRBC/OtBzAN67u7J9YjrOFk7Mb3WdCp7Z405tFWXN8OfPXTDA/JVhc9Xgb2bqcsS4r2cu/uUzgGBPI5JJG92B5Z9VZn8ZjgbQFoxuHHNnTs3Z8+eZc2aNZw7d47o6Gi6dOlCu3btkt2sJYQQIm29HCww138uRbMXNXVZ6cL34S4sTy8HFPBrDC0WgnXWfbtUmIcD1x7SY8VJYhM1lPBxIaBzJTyd5ev6Td7pfRQrKyvat2+f1rUIIYR4jTPhZ+izpw+RCZHkd8nP3LpzyeWUy9RlGZ+iYLHvZ0rfXaZbrthFl3xlYWnauoRIpcfRCQTejgCUZOtDnsQxYdsVkrQK1QrlYG6HCjjZyvCWt0nVZ2jz5s00bNgQa2trNm/e/MZtmzRpkiaFCSGE0HkxWKBUjlLMqjOLbHbZ3v7EzE6TBH/1x/LMCt1ijRFY1hwGWWHWBGEWTgVH0GVJIBGx6tdu06SMD5M+K4ONlcEzlGZJqWpcmzVrRlhYGJ6enjRr1uy126lUqlfOOCCEEOLdvBgs8HGuj5lUY1LWmKM1MQbWd4br21FUFpzJ3YmS1b7BUppWkUnsufKAXitPEa/WkjubPV6vmAmgVjFPetYoiIWFfF2nVqoaV61W+8r/p6VffvmFESNG0L9/f6ZNmwZAfHw833zzDWvWrCEhIYH69esze/ZscubMehPuCiGyFkVRWHB+Ab+d/g2ApgWbMuqjUVhbZIE7imMew6rWcO8EWNmhab6Q4BtaSpq6LiFSad2JEEZsOo9Gq1CzqAez2pbHUYYBpIkMcV06MDCQefPmUbp06WTrBw4cyF9//cX69evZv38/oaGhtGjRwkRVCiFE+tBoNYw7Nk7ftHYr1Y0fq/6YNZrWiDu61Kt7J8A+G3y5GaVIFpnqS2R6iqIwa+8Nhm44h0ar0LJ8bhZ8WVGa1jSUqs/kjBkzUr3Dfv36GVRAdHQ07dq1Y8GCBfz000/69ZGRkSxatIhVq1ZRu3ZtAAICAvDz8+Off/7hww8/NOg4QgiRGbwcLDC88nDa+rU1dVnp48VgAZfc0GETeBQF9evHBwqRUWi0CmP+usiyo3cA6FmzIEPrF80aSXbpKFWN69SpU1O1M5VKZXDj2rt3bz755BP8/f2TNa4nT55ErVbj7++vX1esWDHy5s3L0aNHX9u4JiQkkJCQoF+OitIlUKjVatTp8MPv+THS41jCeOQ8mofMdh6fJT5j4IGBnAo/hbWFNeM+God/Xv9MU//7UN05hOX6DqgSnqF4+JH0+Tpw8YYXfnZnhc+DOTPn85ig1jB44wW2XXyASgXfNSxKxyr5SEpKMnVpacqY5zC1+0xV4xoUFPRexbzOmjVrOHXqFIGBgSkeCwsLw8bGBjc3t2Trc+bMSVjY62P+xo8fz5gxY1Ks37FjBw4O6XdDw86dO9PtWMJ45Dyah8xwHqO0USyNXsoD7QNssaWdfTsSLySy5cIWU5dmdD4Rxyl/Zy4qJYlHjkU55tWPpEOngdPJtssM51G8nbmdx9gkWHTVkhtRKixVCu0LafGIuMiWLRdNXZrRGOMcxsbGpmo7kw26CAkJoX///uzcuRM7u7SbbHfEiBEMGjRIvxwVFUWePHmoV68eLi4uaXac11Gr1ezcuZO6detibZ0FxqOZKTmP5iGznMdbkbfos7cPD7QPyGGfg5k1Z1IkWxFTl5UuLAIXYHF6FioUtEU/xbXZXOpZJf+dkFnOo3gzczyPYVHxdF12ihtR0TjaWjKnbVmqFHA3dVlGY8xz+Pwd8rd5p8b17t27bN68meDgYBITE5M9NmXKlFTt4+TJk4SHh1O+/H850xqNhgMHDjBz5ky2b99OYmIiT58+TXbV9cGDB3h5eb12v7a2ttja2qZYb21tna7fKOl9PGEcch7NQ0Y+j2fCz9B7d2+iEqPI75KfeXXn4ePkY+qyjE9RYPdYOPTv74yKXbBo9CsWbwgWyMjnUaSeuZzHG+HRdFwcyL2ncXg427KkcyVK+Liauqx0YYxzmNr9Gdy47t69myZNmlCgQAGuXLlCyZIluX37NoqiJGtC36ZOnTqcP38+2brOnTtTrFgxhg0bRp48ebC2tmb37t20bNkSgKtXrxIcHEyVKlUMLVsIITKcfSH7GLJ/CPGaeErnKM3MOjOzSLCAGv7qD2dW6pZrfw8fD5ZgAZFpnLwTQZelgTyNVeObw5FlX1UmT/YsML9yBmBw4zpixAgGDx7MmDFjcHZ2ZuPGjXh6etKuXTsaNEj9lCXOzs6ULJl8Vj5HR0fc3d3167t06cKgQYPInj07Li4u9O3blypVqsiMAkKITG/T9U2MOToGraLNgsECneD6DlBZQOPpUP5LU1clRKrtvvyA3qt0wQJl8rixuGNF3J1SvtMrjMPgxvXy5cusXr1a92QrK+Li4nBycmLs2LE0bdqUnj17pllxU6dOxcLCgpYtWyYLIBBCiMxKURTmn5vPzDMzgawYLPAZ3DsJVvbwWQAUbWjqqoRItXWBIYz4/b9ggdntyuNgI3O0pieDP9uOjo76ca3e3t7cvHmTEiVKAPDo0aP3Kmbfvn3Jlu3s7Jg1axazZs16r/0KIURGoNFqGH98PGuvrgV0wQJ9y/XNGvM8RtzWzdH6+IYuWKDtOshT2dRVCZEqz4MFJu24BkDL8rn5pWUprC0zRI5TlmJw4/rhhx9y6NAh/Pz8aNSoEd988w3nz59n06ZN8ha+EEK8RoImgeEHhrMreFcWDBY4/2+wwANwzQPtN+qCBYTIBDRahdGbL7L8H12wQK+aBRkiwQImY3DjOmXKFKKjowEYM2YM0dHRrF27lsKFC6d6RgEhhMhKohKj6Lu7rz5Y4JePf6Fe/nqmLit9BB2ANe0gIQo8i+uaVpcsMGuCMAvxag0D155h64UwVCoY9WlxOlX1NXVZWZrBjWuBAgX0/3d0dGTu3LlpWpAQQmRGSdokfg38lc03N6NRNCkeU2vVOFk7MaP2DCp5VTJRlens4u+wqTtoEiFfVfh8Fdi7mboqYUY0WoUJ266w+ngwGq2S5vtP0iokJmmxsbRgSpsyfFpa/ugytfcaURwdHY1Wq022Lj0m+RdCiIwkLimOIfuHsP/u/tdu4+XoxczaMymaPYu8RX5sHmwdBijg1xhaLATrtAubESJerWHQujNsOf/6NM204GpvzZz25fmoYA6jHkekjsGNa1BQEH369GHfvn3Ex8fr1yuKgkqlQqPRvOHZQghhXp7GP6X3nt6ce3gOW0tbfqr6EyVzlEyxXU7HnFlj5gBFgd1j4NBU3XKlrtBwIrwhWEAIQ0XGqem+7ATHgp5gY2nBLy1LUSl/dqMcy8PZFjtr+frNKAxuXNu3b4+iKCxevJicOXPK4GQhRJYVGh3K1zu/5nbUbVxsXJhZZyblPMuZuizT0ahhcz84u0q3LMECwggeRMXTcfFxroQ9w8nWivkdKvBRIbkamlUY3LiePXuWkydPUrRoFnm7SwghXuFaxDV67uxJeFw4Xo5ezPWfS0G3gqYuy3QSY2BdR7ixE1SW/wYLdDB1VcLM6GJWj2fJmFWhY3DjWqlSJUJCQqRxFUKYvcdxjxl3bBy3nt5K8VhoTChxSXEUcivEHP85eDl6maDCDCLmEaxq/UKwwBIomvokRSFe9DQ2kR/+vMiV+1EpHgt9GkdMooYCORxZKjGrWZLBjevChQvp0aMH9+7do2TJklhbJx+zVbp06TQrTgghTCXkWQg9dvYg+Fnwa7cp71meGbVn4Gqbha/4RNyG5S3gyU0JFhDvLfRpHF8uPs6N8OjXbiMxq1mbwY3rw4cPuXnzJp07d9avU6lUcnOWEMJsXHp8iV67evE4/jG5nHLx7QffYm9ln2wbG0sbSrqXxDIr33R0/xysbPVCsMAm8Chi6qpEJnU17BkdFx8nLCoeLxc7fmpWEkfb5G2KjZWKMrndsJLEqizL4Mb1q6++oly5cqxevVpuzhJCmJ2joUcZsHcAsUmxFMtejNl1ZuPh4GHqsjKeW/t1wQKJz8CzxL/BAt6mrkpkUsduPabbshNExSdR2NOJpV9VxsfN/u1PFFmOwY3rnTt32Lx5M4UKFTJGPUIIYTJbbm3hu8PfkaRN4gOvD5hWaxpONk6mLivjubAJfv/632CBavD5SgkWEO9s24Uw+q05TWKSlor5srGwY0XcHGxMXZbIoAxuXGvXrs3Zs2elcRVCmJXll5YzMXAiAA3yN2BctXHYWMovzxT+mQvbhgMKFG8KzedLsIDQi05IYtL2q9x+HJPiMUWrEP7Qgk2PTqGy0L1bq9EqHL7xCK0CdYvn5LcvysmcqeKNDG5cGzduzMCBAzl//jylSpVKcXNWkyZN0qw4IYQwNq2iZdrJaQRcDACgvV97hlQagoVKxtAlkyJYoBs0nCDBAkLv4bMEOi85zoV7KWcD+I8Fl58+SrH2i8p5+bFpCRm7Kt7K4Ma1R48eAIwdOzbFY3JzlhAiM1Fr1Iw8MpK/b/0NwIDyA/iq5Fcydv9lGjVs7gtnV+uWJVhAvOT2oxi+XHyc4CexuDvaMLBuEWytkjehGo2Gc+fOUbp0aSwt//uDx9vVnqqF3OX7TqSKwY2rVqs1Rh1CCJGuYtWxDNw3kCOhR7BSWTGm6hiaFJR3jFJIiIb1HeHGLl2wQJMZUK69qasSGci5u0/pHBDI45hE8mZ3YNlXlcmfwzHFdmq1GvuwszQqnyvFu7VCpJbBjasQQmR2j+Me03t3by4+voi9lT1Tak6hWq5qpi4r44l5BCs/g9BTumCB1kuhSH1TVyUykP3XHtJzxUliEzWUzOVCQKfKeDjL/KrCeN6pcd2/fz+TJk3i8uXLABQvXpwhQ4bw8ccfp2lxQgiR1kKiQvh619eEPAshm202ZtWZRSmPUqYuK+NJFiyQHdqth9wVTV2VSCNbzt/nyM2UY00NkZikZdOpeyRpFaoVysHcDhVwspXrYcK4DP4KW7FiBZ07d6ZFixb069cPgMOHD1OnTh2WLFlC27Zt07xIIYRIC5ceX6Lnrp48iX9CLqdczKs7j3wu+UxdVsZz/yysaAUx4eCaFzpsghyFTV2VSAOKojBx+1Xm7LuZZvtsWtaHX1uVwcZKbqwSxmdw4zpu3DgmTpzIwIED9ev69evHlClT+PHHH6VxFUJkSBIskEq39sGa9rpggZwlod0GCRYwE2qNluEbz7Px1F0Avqich5wu7zeVWX53R5qU8cHCQm6sEunD4Mb11q1bNG7cOMX6Jk2a8O2336ZJUUIIkZa23t7KqH9GSbDA25zfAL/3AK0a8n+sCxawczV1VSINxCYm0WvlKfZdfYilhYrxLUrRumIeU5clhMEMblzz5MnD7t27UwQQ7Nq1izx55JtACJGxHI4/zNYjWwFomL8hP1X7SYIFXuWfOf8GCwDFm0GL+WAlN9mYg8fRCXy1JJCzdyOxs7Zgdrvy1C6W09RlCfFODG5cv/nmG/r168eZM2f46KOPAN0Y1yVLljB9+vQ0L1AIId7m4qOL/HP/nxTrbz69ydZ4XdMqwQKvoSiwaxQc/vfnd+Xu0OAXCRbIAEKexLLtQhhJWuWd96GgsP7EXYIexeDmYM3iTpUonzdbGlYpRPoyuHHt2bMnXl5eTJ48mXXr1gHg5+fH2rVradq0aZoXKIQQb/LHjT8YfWQ0GuX14Sf9yvaja+muMsH5y14OFqgzEqoNkmCBDOB40BO6Lg0kKj4pTfaXy82epV9VppCnDJERmds7zVvRvHlzmjdvnta1CCFEqimKwqILi5h+Snel8AOvD/B2Sn4TkaJVcH7gTKfinaRpfVlCNKz7Em7ulmCBDGb7xTD6rj5NYpKW4t4ulPBxea/9uTlY0/XjAu99I5YQGcE7T7iWmJhIeHh4iiStvHnzvndRQgjxJhqthomBE1l1ZRUAnUt2ZkD5ASmGAajVarZs2WKKEjO26Iew6jMIPS3BAhnMin/uMPLPC2gV8PfLycy25bCzlmEbQjxncON6/fp1vvrqK44cOZJsvaIoqFQqNJrXv10nhBDvK1GTyIiDI9hxZwcAQysNpUPxDiauKhN5EgQrWsCTWxIskIEoisLUXdeZsfs6oJuq6semJbGylDHZQrzI4Ma1U6dOWFlZ8ffff+Pt7S1vvwkh0s2zxGf039ufwLBArCys+LnazzT0bWjqsjIPCRYwucv3o7geHp1i/YFrD9lwUje/av86hRngX1h+vwrxCgY3rmfOnOHkyZMUK1bMGPUIIcQrhceG03NXT65FXMPR2pFptabxofeHpi4r87i5F9a2h8RoyFlKd6VVggXSVcDhIMb+fQnlNZMEWKhgbNOStP9Q0tyEeB2DG9fixYvz6NH75RsLIYQhgiKD6LGzB6ExobjbuTPHfw5+7n6mLivzkGABk3o5ZrVMHjccbZKPW7WxsuDLKvlkflUh3sLgxnXChAkMHTqUn3/+mVKlSmFtbZ3scReX97v7UQghXnTu4Tl67+7N04Sn5HPJx1z/ueR2zm3qsjKPo7Nh+wjd/0s0h+bzJFggHak1WoZtPMemU/cAGFK/KL1qFpRhAEK8I4MbV39/fwDq1KmTbL3cnCWESGsH7h5g8P7BxCXFUdK9JLP8Z5HdLrupy8octFpdsMCRGbrlyl//GywgN/ukl5gEXczq/msSsypEWjG4cd27d68x6hBCiGR+v/47Y46OQaNoqJqrKlNqTMHB2sHUZWUOGjX82QfOrdEt1xkF1QZm6WCBuxGxPIhKSLfjabQK4/53SWJWhUhjBjeuNWrUMEYdQggBpAwWaFKwCaM/Go21hfVbnikAXbDA+o5wY9e/wQK/Qbl2pq7KpBYcuMW4LZdNcuxsDtYskphVIdLMOwcQxMbGEhwcTGJiYrL1pUuXfu+ihBBZk0arYULgBFZf0UWQdi7ZmYHlB8p4wNR6MVjA2gE+WwpF6pm6KpPRahV+3nKZhYeCAF3sqaVF+n0tebva8XOLUhT0kJhVIdKKwY3rw4cP6dy5M1u3bn3l4zLGVQjxLhI0CXx78Ft23NmBChVDKw2lfXGJIE01CRZIJjFJy+D1Z9l8NhSAbxsVo9vHBeSPICEyOYNH6Q8YMICnT59y7Ngx7O3t2bZtG0uXLqVw4cJs3rzZGDUKIczcs8Rn9NzVkx13dmBtYc3E6hOlaTXE/bOwqJ6uaXXLC112Zumm9Vm8mq+WBLL5bChWFiqmtilD9+pyJ78Q5sDgK6579uzhzz//pGLFilhYWJAvXz7q1q2Li4sL48eP55NPPjFGnUIIM/VysMD0WtP5wPsDU5eVedzcC2s7QOIzXbBA+w3g7GXqqtLFk5hE1BptsnXRCUn0W32ai6FRONhYMqd9BWoU8TBRhUKItGZw4xoTE4OnpycA2bJl4+HDhxQpUoRSpUpx6tSpNC9QCGG+XgwWyGGfgzn+cyiWXVL5Uu3FYAHf6tBmRZYIFohXaxi64Zx+GMCruDvaENC5EqVzu6VfYUIIozO4cS1atChXr14lf/78lClThnnz5pE/f37mzp2Lt7fEBwohUkeCBd5TFg0WiIpX033ZCf659QTglTdbFfNyZlbb8uTP4Zje5QkhjMzgxrV///7cv38fgFGjRtGgQQNWrlyJjY0NS5YsSev6hBBm6MDdA3yz7xviNfGUylGKmXVmSrBAar0cLPBBT6j/c5YIFngQFU/X5ae5EvYMJ1sr5neowEeFcpi6LCFEOjK4cW3f/r8bJipUqMCdO3e4cuUKefPmJUcO+QEihHizF4MFquWqxuQakyVYILU0avizN5xbq1v2Hw1VB2SJYIEHcdB6/nFCI+PxcLZlSedKlPAx/2ERQojk3nke1+ccHBwoX758WtQihDBjiqKw8PxCZpzWXSmUYAEDJUTDui/h5m5dsEDTmVC2ramrShenQ54y/YIlMUnx+OZwZNlXlcmTXf7YESIreu/GVQgh3kaj1fDL8V9Yc1UXQdqlZBf6l+8v0xOl1svBAq2XQeG6pq4qXey+/IDeq04Rn6SidC4XAjpXxt3J/MfyCiFeTRpXIYRRJWgSGHFwBDvv7JRggXfx5BYsbwERQVkuWGBdYAgjfj+PRqvg56Zl+VcVcXWUplWIrEwaVyGE0UQlRtF/T39OPDiBtYU1P1f7mQa+DUxdVuYRegZWtoKYh7pggfa/Q45Cpq7K6BRFYdbeG0zacQ2A5uV8+NgmGAcb+ZUlRFZn/rehCiFMIjw2nE7bOnHiwQkcrR2Z4z9HmlZD3NwLSz7RNa1epXRpWFmgadVoFUZtvqhvWnvWLMiE5iWwlN9WQgjesXE9ePAg7du3p0qVKty7dw+A5cuXc+jQoTQtTgiROd2KvEX7Le25HnGdHPY5WNJgiaRhGeLcelj5GSRG64IFOm3JEmlY8WoNfVadYtnRO6hUMKpxcYY1KCZjoYUQegY3rhs3bqR+/frY29tz+vRpEhISAIiMjOTnn39O8wKFEJnL2Ydn+XLrl9yPuU8+l3wsb7hc0rAMcXQWbOqqS8Mq0QLabQA7F1NXZXSRcWo6Lj7O1gth2Fha8NsX5ehc1dfUZQkhMhiDG9effvqJuXPnsmDBAqyt/5vGpmrVqhL5KkQWtz9kP123dyUyIZJSOUqxrOEyScNKLa0WdnwP27/VLX/QE1ouyhJpWGGR8bSZd5RjQU9wsrViSedKfFrax9RlCSEyIINHul+9epXq1aunWO/q6srTp0/ToiYhRCYkwQLvISlRFyxwfp1u2X8MVO2fJYIFboQ/o+PiQO49jZNgASHEWxncuHp5eXHjxg3y58+fbP2hQ4coUKBAWtUlhMgkFEVh/rn5zDwzE4CmBZsy6qNREiyQWgnP/g0W2AMWVtBkJpT9wtRVpYuTdyLosjSQp7FqCRYQQqSKwY1rt27d6N+/P4sXL0alUhEaGsrRo0cZPHgwP/zwgzFqFEJkUBqthvHHx7P2qi6CtGuprvQr109upkmt6HDdTVj3z/wbLLAcCvubuqp0oQ8WUGspk8eNxR0rSrCAEOKtDG5chw8fjlarpU6dOsTGxlK9enVsbW0ZPHgwffv2NUaNQogM6OVggWGVh9HOr52py8p4khJg7zjdnKwve3QdnoWCgzu0XQ+5K6R7eYZQFIU1gSFsOX8fraK88360Wjh++wkarULNoh7Mblde5mgVQqSKwT8pVCoV3333HUOGDOHGjRtER0dTvHhxnJycjFGfECIDikqMot+efpx8cFIXLPDxzzTIL3O0phAfCWvawe2Dr9/GLR90+B3cC6ZfXe9Aq1UYt+Uyiw4Fpdk+W1XIzfgWpbCWSVqFEKn0zn/i2tjYULx48bSsRQiRCTyIeUDP3T25HnEdJ2snpteaTmXvyqYuK+N5FgYrWsGD82DjDP6jwD5b8m0srKBATbB3M0WFqZaQpGHw+nP8dTYUgN61ClIkp/N77dPD2ZYqBdxlWIkQwiAGN661atV64w+aPXv2vFdBQoiM69bTW/TY1YP7MffxsPdgjv8cimYvauqyMp5H12F5C4gMBkdPaL8BvMuYuqp38ixeTY8VJzl84zFWFiomfVaGZuVymbosIUQWZXDjWrZs2WTLarWaM2fOcOHCBTp27JhWdQkhMpgz4Wfos6cPkQmR5HfJz9y6c8nlJA1MCndP6G64insC2QtC+42QPXNOpB/+LJ7OAYFcDI3CwcaSue0rUL2Ih6nLEkJkYQY3rlOnTn3l+tGjRxMdHf3eBQkhMp79IfsZvH8w8Zp4Sucozcw6M8lml+3tT8xqru2A9R1BHQs+5aHdenDMYbTDRSckMf/ALcIi44yy/yM3H3M3Ig53RxsCOleidG43oxxHCCFSK81u42zfvj2VK1dm0qRJabVLIUQG8GKwwMe5PmZSjUkSLPAqp1fC5r6gaKCQP3y2FGyNd9Pqw2cJdF5ynAv3oox2DIB87g4s7VyZ/DkcjXocIYRIjTRrXI8ePYqdnV1a7U4IYWISLJBKigKHpsDusbrl0p9D05lgabzP0+1HMXy5+DjBT2Jxd7Sh00f5sbBI+5ucHGwsaVo2F9kdbdJ830II8S4MblxbtGiRbFlRFO7fv8+JEyckgEAIM/FysEC3Ut3oW66v3AH+Mq0Gto2A4/N0y1UHgP9oo0a1nrv7lM4BgTyOSSRvdgeWfSVXQ4UQWYfBjaura/IMaQsLC4oWLcrYsWOpV69emhUmhDCNBE0Cww8MZ1fwLlSoGF55OG392pq6rIwnKQE2dYdLfwAqaDAePuxp1EMeuPaQHitOEpuooYSPCwGdK+HpLO90CSGyDoMaV41GQ+fOnSlVqhTZssmNGUJkBpcfX+bA3QMopC7p6EjoEU6Hn8bawppfPv6FevnlD9IUXgwWsLCGFvOgZMs02XXIk1j+PncftUabbH10QhKLDwWRpFWoVigHcztUwMlW0qaEEFmLQT/1LC0tqVevHpcvX5bGVYhMYGvQVr499C1J2iSDnudk7cSM2jOo5FXJSJVlYlH3YWUreHBBFyzw+UooUCNNdn0qOIIuSwKJiFW/dpsmZXyY9FkZbKwkbUoIkfUY/Od6yZIluXXrFr6+mXNeQiGyihWXVjAhcAIAlb0qk88lX6qeZ2tpS6sirSjolrEjSE3ixWABp5zQbgN4l06TXe+58oBeK08Rr9bi5+1CubxuKbYp7u1C28p5jXIjlhBCZAYGN64//fQTgwcP5scff6RChQo4Oia/KcDFxSXNihNCGE5RFKaemkrAhQAAvij2BcMqDcPSwtLElWVyLwcLdNgE2fKnya7XBYYw4vfzaLQKNYt6MLtdeRxsZBiAEEK8zOCfjI0aNQKgSZMmye4wVhQFlUqFRqNJu+qEEAZRa9WMPjKazTc3A9C/fH+6lOwiswG8r2vbYV1HSIpL02ABRVGYtfcGk3ZcA6Bl+dz80rIU1pYyDEAIIV7F4MZ17969xqhDCPGeYtWxDNo3iMOhh7FUWTL6o9E0K9TM1GVlfqdXwOZ+7xUscPl+FJdCUwYFHA96wtoTIQD0rFmQofWLyh8ZQgjxBgY3rr6+vuTJkyfFD1dFUQgJCUmzwoQQqfck/gm9d/XmwuML2FvZM6nGJKrnrm7qsjI3RYGDk2HPj7rlMl9Ak98MDhZY8c8dRv55Ae1rJnVQqWDkp8XpXFXuGxBCiLd5p8b1/v37eHp6Jlv/5MkTfH19ZaiAEOks5FkIPXb2IPhZMG62bsyqM4vSHmlzw1CWpdXA1mEQuEC3XLU/+I8xKFhAURSm7rrOjN3XASiX1w0Xu+RNr7WlijaV8lK3eM40K10IIcyZwY3r87GsL4uOjpbIVyHS2eXHl+m5qyeP4x+TyykXc/zn4OsqV+7eizoefu8Ol/7kXYMFkjRavv/jAmsCde9C9a9TmAH+hWUYgBBCvKdUN66DBg0CQKVS8cMPP+Dg4KB/TKPRcOzYMcqWLZvmBQohXu1o6FEG7htIjDqGotmKMsd/Dh4OHqYuK3OLj4TVbeHOIbC0geZzDQ4WiEvU0Hf1aXZdfoCFCsY2LUn7D1M3FZkQQog3S3Xjevr0aUB3xfX8+fPY2NjoH7OxsaFMmTIMHjw47SsUIgt7lviMK0+upFh/6+ktfgn8hSRtEpW9KjOt1jScbZxNUKEZSYNggaexiXRZeoKTdyKwsbJgxuflaFDSy0gFCyFE1pPqxvX5bAKdO3dm+vTpMl+rEEZ24dEFeu3qRURCxGu3qZ+/Pj9X+xkbS5vXbiNS4eE1WNHyvYIF7j2No+Pi49wIj8bFzopFnSpRKX92IxUshBBZk8FjXAMCAoxRhxDiBYfuHWLQvkHEJcXhbueOq61rssdVqKibvy49y/TEQiVzfr6XkEBY1fq9ggWuhj2j4+LjhEXF4+Vix9KvKlPUS66ACyFEWpNoFiEymM03NzPq8CiSlCQ+8vmIKTWn4Gjt+PYnCsO9GCyQqwK0XWdwsMCxW4/puuwEz+KTKOzpxNKvKuPjZm+kgoUQImuTxlWIDEJRFAIuBjD15FQAPinwCT9+9CPWBs4bKlLp1HL4q/+/wQJ1ofVSsDHsD4RtF8Lot+Y0iUlaKubLxsKOFXFzkGEbQghhLNK4CpEBaBUtvwb+yorLKwDoVKITAysMlGEAxqAocHAS7PlJt1ymLTSZ8V7BAnWL5+S3L8phZ21phIKFEEI8J42rECaWqEnku0Pfse32NgAGVxxMxxIdTVyVmXo5WKDaIKgz8r2CBb6onJcfm5bAylL+yBBCCGOTxlUIE4pOjGbA3gEcCzuGlYUV46qOo1GBRqYuyzylCBb4BT7sYdAuXg4WGOBfmP51JFhACCHSizSuQpjIo7hH9NzVkytPruBg5cC0WtOo4lPF1GWZp/hI2NDxhWCBeVCyhUG7eDlY4KdmpWj7QV4jFSyEEOJVpHEVwgRuR96mx64e3Iu+R3a77Mzxn0Nx9+KmLsss2akjsFreGMIvga2LLljAt7pB+3gxWMDWyoIZX5SjfgkJFhBCiPQmjasQ6ez8w/P03t2biIQI8jrnZa7/XPK45DF1Webp0XU+vjoWlfqxLlig/UbwKmXQLiRYQAghMg5pXIVIRwfvHuSb/d8QlxRHcffizK4zG3d7d1OXZZ5CjmO1qjXW6giU7AVRdfgdsuUzaBdXwqLouPg4D6IS8HbVBQsUySnBAkIIYSrSuAqRTv688SejjoxCo2j4yOcjptacioO1g6nLMk9Xt8H6TqiS4njiUBDnjluwdjXsrX0JFhBCiIxHGlchjExRFBZdWMT0U9MB+LTAp4z9aKwECxjLC8EC2oL+HHFsQ30Hw65qb7twn35rzpCYpKVS/mws/LISrg5yvoQQwtRk4kEhjEiraJkQOEHftHYu0Zlx1cZJ02oMigL7f4XNfXRpWGXboflsORpLW4N2s/yfO/RceYrEJC31iudkeZcPpGkVQogMQq64CmEkiZpEvj30LdtvbwdgSMUhfFniSxNXZaa0Gtg6FAIX6pY//gZq/wBJSanehaIoTNl5jd/23ACg7Qd5+bFpSSwtZI5WIYTIKKRxFcIIniU+Y8DeARwPOy7BAsamjodN3eDyZkAFDSfAB18btIskjZbvfr/A2hO6YIGB/kXoV6eQBAsIIUQGI42rEGnsYexDeu7qydWIqxIsYGxxT2FNW7hz+L2CBfqsOsXuK+ESLCCEEBmcSce4jh8/nkqVKuHs7IynpyfNmjXj6tWrybaJj4+nd+/euLu74+TkRMuWLXnw4IGJKhbizW5H3qbD1g5cjbiKu507SxoskabVWKJCIaChrmm1ddHN0Wpg0xoRk0i7hf+w+0o4tlYWzG1fQZpWIYTIwEzauO7fv5/evXvzzz//sHPnTtRqNfXq1SMmJka/zcCBA/nrr79Yv349+/fvJzQ0lBYtDPvlJER6OPfwHF9u/ZJ70ffI65yX5Y2W4+fuZ+qyzNPDq7Cwri4Ny8kLOm8xOA3rbkQsreYe4VTwU1ztrVnZ9QPqSRqWEEJkaCYdKrBt27Zky0uWLMHT05OTJ09SvXp1IiMjWbRoEatWraJ27doABAQE4Ofnxz///MOHH35oirKFSOHFYIES7iWYVWeWBAsYS/AxWNUa4p+CeyFov0mCBYQQIovIUGNcIyMjAcieXRenePLkSdRqNf7+/vptihUrRt68eTl69OgrG9eEhAQSEhL0y1FRUQCo1WrUarUxy9cf58V/ReZkyHn869ZfjD02Fo2ioYp3FX6t9isOVg7yNWAEqmvbsPy9G6qkOLQ+FdC0WQUO7vCKz/Xj6ATGb73KhSBLlt07luxGqyth0UQnJFHY05FFX1bA29VOzlcGJj9XzYOcx8zPmOcwtftUKYqipPnR34FWq6VJkyY8ffqUQ4cOAbBq1So6d+6crBEFqFy5MrVq1WLChAkp9jN69GjGjBmTYv2qVatwcJCUIpF2FEXhQMIBdsbvBKCsdVmaOzTHUmVp4srMU95H+ygbEoAKhTCXMpzI3+e1c7Q+ioc5ly15FP/6WQEKOCt0K6bBIUP9+S6EEFlTbGwsbdu2JTIyEhcXl9dul2F+ZPfu3ZsLFy7om9Z3NWLECAYNGqRfjoqKIk+ePNSrV++Nn4i0olar2blzJ3Xr1sXaWiYtz6zedh61ipZJJyex85quae3o15F+ZfvJ9EnGoChYHJ6C5enFAGhLt8W90WTqvybE4WJoFD8uP8Wj+ERyudlRxyOG8mVLY2n53487BxtLPvTNjo2VZLBkBvJz1TzIecz8jHkOn79D/jYZonHt06cPf//9NwcOHCB37tz69V5eXiQmJvL06VPc3Nz06x88eICX16tvorC1tcXWNuVVGGtr63T9Rknv4wnjeNV5TNQk8t2h7/TBAkMrDaVD8Q6mKM/8aTWwZTCc0DWtfDwYi9rfY/GaPxAOXX/E18tPEJOowc/bhYUdynHi4G4alc4l349mQH6umgc5j5mfMc5havdn0ssNiqLQp08ffv/9d/bs2YOvr2+yxytUqIC1tTW7d+/Wr7t69SrBwcFUqSJTDIn09yzxGT129WD77e1YWVgxsfpEaVqNRR0P6778t2lVQcNfoc4P8Jqm9c8z9+i85DgxiRo+KujO2q8/xNPZsLhXIYQQGZtJr7j27t2bVatW8eeff+Ls7ExYWBgArq6u2Nvb4+rqSpcuXRg0aBDZs2fHxcWFvn37UqVKFZlRQKS78Nhweu7qybWIazhaOzKt1jQ+9JavQ6OIi4DVbSH4iC5YoMUCKNHstZsvPHiLn/53GYBPSnszpXUZbK0s5SYQIYQwMyZtXOfMmQNAzZo1k60PCAigU6dOAEydOhULCwtatmxJQkIC9evXZ/bs2elcqcjqgiKD6LGzB6ExobjbuTPHf47M0WoskfdgZSvdHK22Lvy/vfuOjqJu2zj+3WwqgSR0CKEElC5Il6KgoGBHUBFBqgWBRyAoRcX6KqCiWBAUhKgIPCooSlOkCQhJCEVKKNJCSQg9Cem7v/ePlTxEggZJstnN9TmHc5zZ2Zl7vYHczM7MxSNzIPTmXDe12w0Tlu3m018PANCvTQ1euqc+Hh661lhExB05dXDNywMNfH19mTJlClOmTCmEikQu9/vJ3xmyYgjn0s9RrVQ1pt0+jaqlqjq7LPeUsBtmd4fEo45ggd7zoVLDXDfNyLIzev7vfLflGACjutTh6fa1dIOciIgbKxI3Z4kUVWuPrWXM+jGkZqXSsGxDpnSaQhnfMs4uyz3lCBa4Hh5bAEG5x69eSM9i0Oxo1u47hdXDwsTujXiwWUiu24qIiPvQ4CpyBdHp0fzw6w/YjI22Vdrybvt3KeGlZwEXiN1L4Nv+kJUGIS3g0a+hRO7/QDiVnM6A8Ch+P3oePy8rH/duyq11KhRywSIi4gwaXEX+whjDzJ0z+S71OwDuq3Ufr7R5BS8PPb6lQER/DouGg7HD9Z3hoVmke/jy9YZDnEhMv2zzRb8f59DpFMr4ezOzXwturBpU6CWLiIhzaHAVuYTNbmNi1ETm7p4LQL/6/QhrHqbrJguCMfDr27DqDcdyk95wz/skZhqeCo9iw4HTV3xrSGk/vhjQkprlSxZSsSIiUhRocBX5U7otnefXPs/Ph38G4C7fu5SGVVByCRbgthdJSEqn76woYuIS8fe20q1pCNa/PCEgwNeT3q2rU6GUrxMKFxERZ9LgKoIjWGDYqmFExUfh6eHJ6ze9jm2XzdlluafMVJj/OOxeBFjgrreh5RMcOJlMn5mRHD2bSrmSPoT3b0HDKoHOrlZERIoQBXVLsZeQkkC/Zf2Iio/C38ufaZ2m0blGZ2eX5Z5Sz8KX3RxDq9UbHv4cWj7BltizPDhtA0fPplKjbAkWPN1GQ6uIiFxGZ1ylWLtSsIASlwrA+WOOZ7SejAGfQOg5B2q0Y9WeBAbP3kxqpo1GIYHM7NeCciUV1SoiIpfT4CrF1raT2xi6Yijn0s9RPaA60zpNI6SUngVaIBJ2w+xukHgMSlV2BAtUbMC30UcZPf93bHbDLbXLM7VXU/x99NeSiIjkTj8hpFj69eivjFw9kjRbmoIFClrsRpjTwxEsUK429J6PCazK1NV/8NayPQB0a1KFiQ82wsuqq5dEROTKNLhKsfPdvu94dcOrChYoDLsXw7cDcgQL2H1L89qPuwj/7RAAT7WvyZgudfX0BhER+UcaXKXYMMYwfft0PtzyIaBggQIXHQ6LRjiCBWp3gQdnke7hQ9i8LSz+PQ6AcffUZ2C7UOfWKSIiLkODqxQLNruNCZETmLdnHgADGg5geNPhOstXEIyBNW/B6jcdy5cECzwZHsnGA2fwslqY9PCN3Nc42Lm1ioiIS9HgKm4v3ZbO2LVjWX54ORYsjGoxit71ezu7LPdkt8HikRA9y7F8y3Nw6wucSEqn78xIdscnUdLHk08ea0bb68o5t1YREXE5GlzFrSVmJDJs5TA2ndiEl4cXb7Z7ky6hXZxdlnu6QrDA/pPJ9PkskmPnFCwgIiLXRoOruK2ElAQG/TKIfWf34e/lz/u3vk+ryq2cXZZ7SjkDc3vCkY1g9YHu06H+/WyJPcuA8CjOpmRSo2wJvhjQimpldSOciIj8OxpcxS0dOH+AQcsHEXchjnJ+5ZjaaSp1y9R1dlnu6fzRP4MFducMFtidwOCvHMECjUMC+UzBAiIico00uIrb2XZyG0NWDOF8+nkFCxSw9OM7sM55EM/kOLL8KxF372wy/euxMSKWcQt3YLMb2tcuz8cKFhARkXygnyTiVtYcWcOza54lzZbGDeVu4KOOHylYoIDERPxM8NJ+BHKBP+zB9D09mmPhCUBC9jbdmlZhYncFC4iISP7Q4Cpu49JggXZV2jGp/SQFCxSQLT/Ppt764fhaMtliavOMx2iSfEsR8OfrXlYPet1UnRGdrtcjx0REJN9ocBWXp2CBwhXxzSSa73gdq8WwtURr6g75hrX+pZxdloiIFAMaXMWl/TVY4PEbHueZJs/oLF8BMHY7G8NH0zr2U7BAVOm7aTI4HE8vb2eXJiIixYQGV3FZfw0WGN1yNL3q9XJ2WS7Dbjdk2u152tbYbGz75HFan1kIwMaQgbQa8A4WD127KiIihUeDq7iky4IFbn6TLjUULJBXO46d5+mvojlyJvUft/Uhgw+8PqKzdRN2YyGqwfPc9PCoQqhSREQkJw2u4nJOXDjB0yuezg4W+ODWD2hZuaWzy3IZa/edZNCX0VzIsP3jtoEkM8P7HVp47CUdL3a1mUSrzn0LoUoREZHLaXAVl3Lg3AEG/fK/YIFpnaZRp0wdZ5flMr7fcoxnv9lGlt3Q9rqyTO7RBF+v3L/utyQew+/rh7Ge2ovxCcT6yByahLYr5IpFRET+R4OruIytCVsZunIo59PPUyOgBtNun0aVklWcXZbLmP7rAd5YEgPAfY2Deeehxnh7XuEa1YQYRxpW4jEoFYyl93w8K9YvxGpFREQup8FVXMKlwQKNyjXio44fUdq3tLPLcgl2u+HNJTHMWHcQgAFtQ3nx7np4eFzhyQuHN8DcHpB2HsrVgd7zIahqIVYsIiKSOw2uUuhSMlN4fePrbDi+Ic/vOZt+Fruxc3OVm3mn/TvFOlhg74kkRn37O0fP/vONVQA2u52zKZkAPH9XXZ64ueaVHxcWswjmD4SsNKjaCnrOgxJKHhMRkaJBg6sUqjNpZxjyyxB2nN5x1e/tel1XXmr9UrEOFth06AwDwqNITMu6qvd5WS1M7N6Ibk1D/mbnM2HxSDB2qH0nPDgTvIvvPxBERKTo0eAqheZo0lEG/TKIw4mHCfIJ4o12b1CxRMU8vdffy5+QUn8zdBUDP+2M55m5W0jPstO0WhCv3tcQT2veghYqBvhSxv8KQQHGwOoJsGaCY7lpH7j7PbDqrwcRESla9JNJCkXM6RgGrxjMqdRTBPsHM+32aYQGhjq7LJcxJyKWF7/fjt1Ax7oV+OjRpvh5W699x7YsWDISosMdy+1HQ4exoOQxEREpgjS4SoGLiItg2KphXMi8QO3StZnaaSoVSlRwdlkuwRjD+yv2MfmXfQD0aF6VNx5oiKc1HxKrMlPh24GwZzFggbsnQYuB175fERGRAqLBVQrUsoPLGLtuLFn2LFpUasH7t75PKe9Szi7LJdjshnELdzAnIhaA/9x2HWG3177yjVVXI+UMzO0JRzaC1Qe6z4D69137fkVERAqQBlcpMLN3zWZi1EQA7qh+B+NvHo+39QrXWUoOaZk2npm7hZ93ncBigdfua8BjrWvkz87PH3U8o/XkbvANdDw5oHqb/Nm3iIhIAdLgKvnOGMN7m99j1o5ZAPSs25PRLUZj9ciHazKLgXMpGTz++SY2HT6Lt6cH7/e4kTtvqJw/Oz+xyzG0Jh2HUsHQ+1uo2CB/9i0iIlLANLhKvsq0Z/LKb6/ww/4fABjWdBgDGw7Mn6+3i4Hj51LpOzOSfQnJlPL1ZEaf5rSqWTZ/dn74N5j7iIIFRETEZWlwlXyTkplC2Oow1h9fj9Vi5ZU2r9D1uq7OLstl7D2RRJ/PIolPTKNSgC/hA1pQt1JA/uw85kfHjVi2dAULiIiIy9LgKvni0mABP08/3mn/DreE3OLsspzGbjeE/3aI6NizeX7P2r0nSUzLolZ5f74Y2IoqQX75U0zUZ7DkWUewQJ27HMECXvm0bxERkUKkwVWu2ZGkIwxaPojYpFiCfIKY0nEKjco3cnZZTpORZee5b7excOvxq35v02pBfNa3BaWvFBZwNYyBVW/Cr2/9ufO+cPe7ChYQERGXpZ9gck1iTsfw9C9PczrttIIFgOT0LAZ9Gc26P07h6WFhcIdalC3pk6f3Bvh5cmfDyvh65VOwwOIRsPkLx7KCBURExA1ocJV/bcPxDYxYPYILmReoU7oOUztNpXyJ8s4uy2lOJqXTPzySHccSKeFt5eNeTelQxwlBCxkpMH8g7FkCFg9HsEDzAYVfh4iISD7T4Cr/ypIDS3hh/Qtk2bNoWaklk2+dXKyDBQ6dukCfmZHEnkmhrL83M/u1oHHVoMIvJOWM48kBRyLA0xe6fwb17in8OkRERAqABle5al/u+pK3ohzXTXau0Zk3271Z5IMFTienMzcylqS0rDxtb7PbOXDYg+0/7cXq8ffxqgZYsPkop5IzqFrGjy8GtCK0nH8+VH2Vzh1xPKP11J4/gwX+C9VbF34dIiIiBUSDq+SZ3diZHD2ZWTsdwQKP1n2U0S1H42H5+8HO2Q6fdpwNPXw65Srf6cHK44fyvHWD4ABm9W9BhVK+V3mcfHBpsEBAFcczWivUK/w6RERECpAGV8mTTHsmL61/iUUHFgGuEyyw/eh5+odHZp8NvbNh3hKobDYbBw8cJLRmKFbrP98sVcbfm16tqlHK1+taS756h9bD3J6Q/mewwGMLIDCk8OsQEREpYBpc5R+5arDA2n0nGfRlNBcybNSvHED4gLyfDc3MzGTJkv3c1aUOXl5OGEbzKkewwE3Qc66CBURExG1pcJW/dTr1NENWDGHn6Z34efoxqf0kbg652dll/aPvtxzj2W+2kWU3tL2uLNN6N3PO2dCCFDUDFj8LGAULiIhIsaDBVa6oIIMF9sQnEXHwdL7s66+OnElh+tqDANzbOJh3HmqEj2c+PBu1qPhrsECzfnDXJAULiIiI29NPOsnVpcECVUpWYVqnadQIrJEv+1641XE2NNNm8mV/VzKgbSgv3l0PD4+ifR3uVbksWGAMdBijYAERESkWNLjKZQoyWGDG2gP83+IYwBFvWimwYO7Av7VOBR5sFlLkbx67Khkp8O0A2LtUwQIiIlIsaXCVHC4NFmhVqRXv3fpevgQL2O2GCct28+mvBwDo37YG4+6u715nQwuSggVEREQ0uMr/fLHzC97e9DaQv8ECGVl2Rn27je+3HgdgzJ11eeqWmu51NrQgKVhAREQE0OAqXB4s0KteL0a1GJUvwQLJ6Vk8PTuatftO4elhYWL3RnRvpmeM5tmJnX8GC8QpWEBERIo9Da7F3F+DBUY0G0H/Bv3z5WzoyaR0BoRHsf3YeUp4W/m4V1M61KlwzfstNi4NFihf1zG0KlhARESKMQ2uxdhfgwVebfMq9193f77s+9KY1TL+3szq14LGVYPyZd/Fwq4fYP7j/wsWeHQe+JV2dlUiIiJOpcG1mCrIYIG/xqx+MaAVoeX882XfxcKlwQJ174HuMxQsICIiggbXYunSYIHSPqWZ0nEKN5S/IV/2fS0xq8WeMbDqDfjVcYMczfo7Hnnl4UbhCSIiItdAg2sxEx23nWGrh3A+4ywV/CrzYrP38LZVY0980jXve+uRs7z4/Q4ybW4cs1pQbFmwaDhs+dKx3GEstB+tYAEREZFLaHAtRj6NWsqHO8aBRzq2tMoc2DuA/psPAYfy9ThuGbNakP4aLHDPe44YVxEREclBg2sx8dqq2Xx9+B0sHjZIvQ6/0wMo4Ze/X+FbPSw81KwqYbfXVrBAXqWcgTk94GikI1jgwZlQ925nVyUiIlIkaXAtBgb/+B5rz8zEYoHyllYs7P8hpXx0s4/TnTsCs7vBqb3gGwSP/heq3eTsqkRERIosDa5uLMtm45H5L7In1fGM1ut87uSbh8bjadVX+E6XI1gg5M9ggbrOrkpERKRI0+Dqpi6kp9P162eIt/8GQOvSfZh2z0g8PK49DUuu0aF1MPfRP4MF6v0ZLFDF2VWJiIgUeRpc3dCJ5PM88M2TJHnswhgPHggZweud+jm7LAHYtRDmP+EIFqjWBnrOUbCAiIhIHmlwdTN7Th7n0R+fIMMai7F7Mbj+awxudY+zyxKAyOmw5DkULCAiIvLvaHB1Ixtj9/DU8kHYPU+BzZ9XW71L9wZtnF2WGAMr/w/WvuNYbj4A7npHwQIiIiJXSYOrm1i4K4IXNw4Hz2Q8ssrycaeptK1ez9lliS0LFg2DLbMdy7e+ALc8p2ABERGRf0GDqxv4NGopH+wYh8WajldWCHPum07d8iHOLksyUuDb/rB3mYIFRERE8oEGVxex/2QyI7/exsFTF3KsN/6bsZebh8XDRkl7Pb57+BMqldLNPk6XcgbmPAxHo/4MFpgFde9ydlUiIiIuTYOrC9gSe5YB4VGcTcnMsd6rzFp8yy/GAlTwuInvH/lAwQJFwblY+LIbnN73Z7DA11CtlbOrEhERcXkaXIu4VbsTGPzVZlIzbTQOCWR8t0Z4ecLnuz9i4cHFADxQ8xFeaTcWD4ue0ep08TscwQLJ8Y5ggccWQPk6zq5KRETELWhwLcK+2XSEMQu2Y7Mb2tcuz8e9muLtaXhx/YssObgEgJHNRtK3QV8sutnH+S4NFqhQH3p9q2ABERGRfKTBtQgyxvDx6v28/dMeALo1rcLE7o3IsKcyZMUINsRtwNPiyWttX+PeWvc6uVoBYOf3sOAJsGUoWEBERKSAaHAtYmx2w+uLdhH+2yEAnu5Qi1Gd63A67TSDfxlMzJkY/Dz9eK/De7St0ta5xYpDxKewdBQKFhARESlYGlyLkLRMGyO/3sbi7XFYLPDSPfXp3zaU2MRYnlr+FEeTj1LGtwxTOk6hYbmGzi5XjIGVr8PaSY7l5gPhrrcVLCAiIlJANLgWEYlpmTz5xSY2HjiDt9WDSQ835t7Gwew8vZPBvwzmTNoZqpSswie3f0L1gOrOLldsmfDjcNh6MVjgRbjlWQULiIiIFCANrkXAicQ0+s6MZHd8EiV9PPn0sWa0ua4cvx37jeGrh5OalUq9MvX4uNPHlPMr5+xyJeMCfNMf9v30Z7DAZGjW19lViYiIuD0Nrk62/2QyfT6L5Ni5VMqX8iG8fwsaBAey6MAixq0bR5bJ4qbKNzH51sn4e/k7u1y5cNoRLHBsE3j6wUOzoM6dzq5KRESkWNDg6kSbY88y8M9ggdBy/nwxoCVVy5Tg852f886mdwC4M/RO3mj7Bl5WLydXK5w97HhG6+l9jicGPPo1VG3p7KpERESKDQ2uTrJy9wkGf7WZtEw7jUMCmdmvBaX9vXg76m2+2PUFAH3q92Fk85EKFigK4rfD7AcdwQKBVaH3fAULiIiIFDINrk7w9aYjjP0zWKBDnfJMedQRLDBm7RiWHlwKOIIF+jXs59xCxeHgWpj3KKQnOoIFes+HgGBnVyUiIlLsaHAtRH8NFujeNIQJ3W8g3ZbC4BUj2Bi3UcECRYwlZiEsfNoRLFC9LTwyB/yCnF2WiIhIsaTBtZDY7IZXf9zJFxsOA1cOFpjcYTJtqrRxcrUCEHpyOdYtswED9e6FbjPAy9fZZYmIiBRbGlwLQVqmjbCvt7Jke/zfBgt83PFjGpRr4OxynScrHaJmwLkjzq4Ea1IcjY5+71hQsICIiEiRoMG1gCWmZfLE55uIOOgIFni3R2PuaRTMzlM7GbzCESwQUjKET27/hGoB1ZxdrvOknYd5veDQWmdXAsDF2+Fs7cdi7TBawQIiIiJFgAbXAnSlYIH1x9YzYvUIBQtclBTvuGP/xHbwLgnNB4CTH/9ls9mJPGGlebuRWDW0ioiIFAkaXAvI/pMXGPjF5suCBX7c/yMvrX9JwQIXnfoDZj8A52LBvwL0+gaCb3R2VdgzM0lYssTZZYiIiMglNLgWgENJ8PL0SM6l/i9YIKS0H+E7wpkUPQlQsAAARzc5UqhSTkOZmtB7AZQJdXZVIiIiUkRpcM1nK/ec5KNdVjLtmTSuGsTMvs0dwQKb3ubLXV8CChYAYN9y+LoPZKZAcBN49BsoWd7ZVYmIiEgRpsE1H+0/mczgOVux2S20v74cUx9rhqfVzphfx7D0kCNY4Nnmz9K3QV8nV+pkW+fAwqFgbFCrIzz8BfiUdHZVIiIiUsRpcM1HtcqX5KmbQ9m06w+m9roRO2kMXjGciLgIPC2evN7ude6peY+zy3QeY2Dde7DiVcdyox5w/xSn34glIiIirsElvqueMmUKNWrUwNfXl1atWhEZGenskq5oeMdaPFrLzvmMMwz4aQARcRGU8CzBlI5TivfQarfDsjH/G1rbDoOu0zS0ioiISJ4V+cH1v//9L2FhYbz88sts3ryZxo0b07lzZxISEpxdWq4sFgun7afo/3N/Ys7EUMa3DDO7zCzeaVhZ6TB/AERMcyx3Hg+3vwYeRf63n4iIiBQhRf5SgXfffZcnnniC/v37AzBt2jQWL17MzJkzGTNmjJOru9yuzZ8yI2kmyWRQ1acM067vR7UT++HEfmeX5jyRnzqCBTy84IFpcMODzq5IREREXFCRHlwzMjKIjo5m7Nix2es8PDzo1KkTGzZsyPU96enppKenZy8nJiYCkJmZSWZmZoHWeyjxEE/smkqqh4V66Rl8fPh3yu0eXqDHdBXGuyS2B7/AhN4CBdyH/HDx90pB/56RgqU+ugf10T2oj66vIHuY130W6cH11KlT2Gw2KlasmGN9xYoV2b17d67vGT9+PK+++upl63/++WdKlChRIHVeZIyhQ7oXJ602XkssicWvFKcL9IiuIdPqz+7K3TgfkwwxrvVQ/+XLlzu7BMkH6qN7UB/dg/ro+gqihykpKXnarkgPrv/G2LFjCQsLy15OTEykatWq3HHHHQQEBBT48W9Lv43lvyynbK878fLSjUcXtXV2AVcpMzOT5cuXc/vtt6uPLkx9dA/qo3tQH11fQfbw4jfk/6RID67lypXDarVy4sSJHOtPnDhBpUqVcn2Pj48PPj4+l6338vIqtD8onhbPQj2eFBz10T2oj+5BfXQP6qPrK4ge5nV/Rfq2bm9vb5o1a8aKFSuy19ntdlasWEHr1q2dWJmIiIiIFLYifcYVICwsjL59+9K8eXNatmzJ5MmTuXDhQvZTBkRERESkeCjyg2uPHj04efIkL730EvHx8dx4440sW7bsshu2RERERMS9FfnBFWDo0KEMHTrU2WWIiIiIiBMV6WtcRUREREQu0uAqIiIiIi5Bg6uIiIiIuAQNriIiIiLiEjS4ioiIiIhL0OAqIiIiIi5Bg6uIiIiIuAQNriIiIiLiEjS4ioiIiIhL0OAqIiIiIi5Bg6uIiIiIuAQNriIiIiLiEjS4ioiIiIhL8HR2AQXNGANAYmJioRwvMzOTlJQUEhMT8fLyKpRjSv5TH92D+uge1Ef3oD66voLs4cU57eLcdiVuP7gmJSUBULVqVSdXIiIiIiJ/JykpicDAwCu+bjH/NNq6OLvdzvHjxylVqhQWi6XAj5eYmEjVqlU5cuQIAQEBBX48KRjqo3tQH92D+uge1EfXV5A9NMaQlJREcHAwHh5XvpLV7c+4enh4EBISUujHDQgI0B9MN6A+ugf10T2oj+5BfXR9BdXDvzvTepFuzhIRERERl6DBVURERERcggbXfObj48PLL7+Mj4+Ps0uRa6A+ugf10T2oj+5BfXR9RaGHbn9zloiIiIi4B51xFRERERGXoMFVRERERFyCBlcRERERcQkaXEVERETEJWhwzWdTpkyhRo0a+Pr60qpVKyIjI51dklzB+PHjadGiBaVKlaJChQp07dqVPXv25NgmLS2NIUOGULZsWUqWLEn37t05ceKEkyqWvJgwYQIWi4Xhw4dnr1MfXcOxY8fo3bs3ZcuWxc/PjxtuuIFNmzZlv26M4aWXXqJy5cr4+fnRqVMn9u3b58SK5a9sNhvjxo0jNDQUPz8/atWqxeuvv54jf159LHp+/fVX7r33XoKDg7FYLHz//fc5Xs9Lz86cOUOvXr0ICAggKCiIgQMHkpycnO+1anDNR//9738JCwvj5ZdfZvPmzTRu3JjOnTuTkJDg7NIkF2vWrGHIkCFs3LiR5cuXk5mZyR133MGFCxeytxkxYgQ//vgj33zzDWvWrOH48eN069bNiVXL34mKiuKTTz6hUaNGOdarj0Xf2bNnadu2LV5eXixdupRdu3YxadIkSpcunb3NW2+9xQcffMC0adOIiIjA39+fzp07k5aW5sTK5VITJ05k6tSpfPTRR8TExDBx4kTeeustPvzww+xt1Mei58KFCzRu3JgpU6bk+npeetarVy927tzJ8uXLWbRoEb/++itPPvlk/hdrJN+0bNnSDBkyJHvZZrOZ4OBgM378eCdWJXmVkJBgALNmzRpjjDHnzp0zXl5e5ptvvsneJiYmxgBmw4YNzipTriApKclcf/31Zvny5aZ9+/Zm2LBhxhj10VWMHj3atGvX7oqv2+12U6lSJfP2229nrzt37pzx8fExc+fOLYwSJQ/uvvtuM2DAgBzrunXrZnr16mWMUR9dAWC+++677OW89GzXrl0GMFFRUdnbLF261FgsFnPs2LF8rU9nXPNJRkYG0dHRdOrUKXudh4cHnTp1YsOGDU6sTPLq/PnzAJQpUwaA6OhoMjMzc/S0bt26VKtWTT0tgoYMGcLdd9+do1+gPrqKH374gebNm/PQQw9RoUIFmjRpwvTp07NfP3jwIPHx8Tn6GBgYSKtWrdTHIqRNmzasWLGCvXv3ArBt2zbWrVvHnXfeCaiPrigvPduwYQNBQUE0b948e5tOnTrh4eFBREREvtbjma97K8ZOnTqFzWajYsWKOdZXrFiR3bt3O6kqySu73c7w4cNp27YtDRs2BCA+Ph5vb2+CgoJybFuxYkXi4+OdUKVcybx589i8eTNRUVGXvaY+uoYDBw4wdepUwsLCeP7554mKiuKZZ57B29ubvn37Zvcqt79j1ceiY8yYMSQmJlK3bl2sVis2m4033niDXr16AaiPLigvPYuPj6dChQo5Xvf09KRMmTL53lcNriI4ztbt2LGDdevWObsUuUpHjhxh2LBhLF++HF9fX2eXI/+S3W6nefPmvPnmmwA0adKEHTt2MG3aNPr27evk6iSvvv76a7766ivmzJlDgwYN2Lp1K8OHDyc4OFh9lHyhSwXySbly5bBarZfdqXzixAkqVarkpKokL4YOHcqiRYtYtWoVISEh2esrVapERkYG586dy7G9elq0REdHk5CQQNOmTfH09MTT05M1a9bwwQcf4OnpScWKFdVHF1C5cmXq16+fY129evWIjY0FyO6V/o4t2p577jnGjBnDI488wg033MBjjz3GiBEjGD9+PKA+uqK89KxSpUqX3YielZXFmTNn8r2vGlzzibe3N82aNWPFihXZ6+x2OytWrKB169ZOrEyuxBjD0KFD+e6771i5ciWhoaE5Xm/WrBleXl45erpnzx5iY2PV0yKkY8eObN++na1bt2b/at68Ob169cr+b/Wx6Gvbtu1lj6Pbu3cv1atXByA0NJRKlSrl6GNiYiIRERHqYxGSkpKCh0fO0cJqtWK32wH10RXlpWetW7fm3LlzREdHZ2+zcuVK7HY7rVq1yt+C8vVWr2Ju3rx5xsfHx4SHh5tdu3aZJ5980gQFBZn4+Hhnlya5ePrpp01gYKBZvXq1iYuLy/6VkpKSvc2gQYNMtWrVzMqVK82mTZtM69atTevWrZ1YteTFpU8VMEZ9dAWRkZHG09PTvPHGG2bfvn3mq6++MiVKlDCzZ8/O3mbChAkmKCjILFy40Pz+++/m/vvvN6GhoSY1NdWJlcul+vbta6pUqWIWLVpkDh48aBYsWGDKlStnRo0alb2N+lj0JCUlmS1btpgtW7YYwLz77rtmy5Yt5vDhw8aYvPWsS5cupkmTJiYiIsKsW7fOXH/99aZnz575XqsG13z24YcfmmrVqhlvb2/TsmVLs3HjRmeXJFcA5Ppr1qxZ2dukpqaawYMHm9KlS5sSJUqYBx54wMTFxTmvaMmTvw6u6qNr+PHHH03Dhg2Nj4+PqVu3rvn0009zvG632824ceNMxYoVjY+Pj+nYsaPZs2ePk6qV3CQmJpphw4aZatWqGV9fX1OzZk3zwgsvmPT09Oxt1MeiZ9WqVbn+POzbt68xJm89O336tOnZs6cpWbKkCQgIMP379zdJSUn5XqvFmEviLEREREREiihd4yoiIiIiLkGDq4iIiIi4BA2uIiIiIuISNLiKiIiIiEvQ4CoiIiIiLkGDq4iIiIi4BA2uIiIiIuISNLiKiIiIiEvQ4CoikgcdOnRg+PDhzi4jmzGGJ598kjJlymCxWNi6detl24SHhxMUFFTotf2Tfv360bVrV2eXISIuSIOriIgLWrZsGeHh4SxatIi4uDgaNmx42TY9evRg79692cuvvPIKN954Y6HVeOjQoVyH6vfff5/w8PBCq0NE3IenswsQESmubDYbFosFD4+rP4ewf/9+KleuTJs2ba64jZ+fH35+ftdSYq4yMjLw9vb+1+8PDAzMx2pEpDjRGVcRcRkdOnTgmWeeYdSoUZQpU4ZKlSrxyiuvZL+e2xm+c+fOYbFYWL16NQCrV6/GYrHw008/0aRJE/z8/LjttttISEhg6dKl1KtXj4CAAB599FFSUlJyHD8rK4uhQ4cSGBhIuXLlGDduHMaY7NfT09N59tlnqVKlCv7+/rRq1Sr7uPC/r+5/+OEH6tevj4+PD7Gxsbl+1jVr1tCyZUt8fHyoXLkyY8aMISsrC3B81f6f//yH2NhYLBYLNWrUyHUfl14qEB4ezquvvsq2bduwWCxYLJbss57nzp3j8ccfp3z58gQEBHDbbbexbdu27P1cPFM7Y8YMQkND8fX1BRxnfdu1a0dQUBBly5blnnvuYf/+/dnvCw0NBaBJkyZYLBY6dOiQXf+llwqkp6fzzDPPUKFCBXx9fWnXrh1RUVHZr1/s2YoVK2jevDklSpSgTZs27NmzJ9fPLSLuS4OriLiUzz//HH9/fyIiInjrrbd47bXXWL58+VXv55VXXuGjjz7it99+48iRIzz88MNMnjyZOXPmsHjxYn7++Wc+/PDDy47t6elJZGQk77//Pu+++y4zZszIfn3o0KFs2LCBefPm8fvvv/PQQw/RpUsX9u3bl71NSkoKEydOZMaMGezcuZMKFSpcVtuxY8e46667aNGiBdu2bWPq1Kl89tln/N///R/g+Kr9tddeIyQkhLi4uBxD3pX06NGDkSNH0qBBA+Li4oiLi6NHjx4APPTQQ9mDe3R0NE2bNqVjx46cOXMm+/1//PEH8+fPZ8GCBdn/MLhw4QJhYWFs2rSJFStW4OHhwQMPPIDdbgcgMjISgF9++YW4uDgWLFiQa22jRo1i/vz5fP7552zevJnrrruOzp075zg+wAsvvMCkSZPYtGkTnp6eDBgw4B8/t4i4GSMi4iLat29v2rVrl2NdixYtzOjRo40xxhw8eNAAZsuWLdmvnz171gBm1apVxhhjVq1aZQDzyy+/ZG8zfvx4A5j9+/dnr3vqqadM586dcxy7Xr16xm63Z68bPXq0qVevnjHGmMOHDxur1WqOHTuWo76OHTuasWPHGmOMmTVrlgHM1q1b//ZzPv/886ZOnTo5jjVlyhRTsmRJY7PZjDHGvPfee6Z69ep/u59Zs2aZwMDA7OWXX37ZNG7cOMc2a9euNQEBASYtLS3H+lq1aplPPvkk+31eXl4mISHhb4938uRJA5jt27cbY3LvhzHG9O3b19x///3GGGOSk5ONl5eX+eqrr7Jfz8jIMMHBweatt94yxuTes8WLFxvApKam/m1NIuJedMZVRFxKo0aNcixXrlyZhISEa9pPxYoVKVGiBDVr1syx7q/7vemmm7BYLNnLrVu3Zt++fdhsNrZv347NZqN27dqULFky+9eaNWtyfH3u7e192Wf4q5iYGFq3bp3jWG3btiU5OZmjR49e9Wf9O9u2bSM5OZmyZcvmqPvgwYM56q5evTrly5fP8d59+/bRs2dPatasSUBAQPYlC1e6/CE3+/fvJzMzk7Zt22av8/LyomXLlsTExOTY9tL/b5UrVwb4V70XEdelm7NExKV4eXnlWLZYLNlfTV+8yclcct1pZmbmP+7HYrH87X7zIjk5GavVSnR0NFarNcdrJUuWzP5vPz+/HAOpsyUnJ1O5cuUc1+JedOmjtPz9/S97/d5776V69epMnz6d4OBg7HY7DRs2JCMjo0Bq/WvPgKvqkYi4Pg2uIuI2Lp4RjIuLo0mTJgC5Pt/034qIiMixvHHjRq6//nqsVitNmjTBZrORkJDAzTfffE3HqVevHvPnz8cYkz2grV+/nlKlShESEvKv9+vt7Y3NZsuxrmnTpsTHx+Pp6XnFm7xyc/r0afbs2cP06dOzP++6desuOx5w2TEvVatWLby9vVm/fj3Vq1cHHP/YiIqKKlLPzRWRokGXCoiI2/Dz8+Omm25iwoQJxMTEsGbNGl588cV8239sbCxhYWHs2bOHuXPn8uGHHzJs2DAAateuTa9evejTpw8LFizg4MGDREZGMn78eBYvXnxVxxk8eDBHjhzhP//5D7t372bhwoW8/PLLhIWF/atHZ11Uo0YNDh48yNatWzl16hTp6el06tSJ1q1b07VrV37++WcOHTrEb7/9xgsvvMCmTZuuuK/SpUtTtmxZPv30U/744w9WrlxJWFhYjm0qVKiAn58fy5Yt48SJE5w/f/6y/fj7+/P000/z3HPPsWzZMnbt2sUTTzxBSkoKAwcO/NefVUTckwZXEXErM2fOJCsri2bNmjF8+PDsO/HzQ58+fUhNTaVly5YMGTKEYcOG8eSTT2a/PmvWLPr06cPIkSOpU6cOXbt2JSoqimrVql3VcapUqcKSJUuIjIykcePGDBo0iIEDB17zEN69e3e6dOnCrbfeSvny5Zk7dy4Wi4UlS5Zwyy230L9/f2rXrs0jjzzC4cOHqVix4hX35eHhwbx584iOjqZhw4aMGDGCt99+O8c2np6efPDBB3zyyScEBwdz//3357qvCRMm0L17dx577DGaNm3KH3/8wU8//UTp0qWv6fOKiPuxmEsvBhMRERERKaJ0xlVEREREXIIGVxERERFxCRpcRURERMQlaHAVEREREZegwVVEREREXIIGVxERERFxCRpcRURERMQlaHAVEREREZegwVVEREREXIIGVxERERFxCRpcRURERMQl/D8v/wD5Pa6JBAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "census\n", + "Running Isolation Forest\n", + "Running AAD Isolation Forest\n", + "Running Pine Forest\n" +======= + "Running Isolation Forest\n" +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + ] + } + ], + "source": [ + "for dataset in DevNetDataset.avialble_datasets:\n", + " print(dataset)\n", +<<<<<<< HEAD + " %time compare = Compare(DevNetDataset(dataset), n_jobs=15).run().plot(dataset, savefig=True)\n", + " plt.show()" + ] +======= + " %time compare = Compare(DevNetDataset(dataset), n_jobs=15).run().plot(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbebe1bb-3b54-4d8f-8624-def1ad6e898e", + "metadata": {}, + "outputs": [], + "source": [] +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", +<<<<<<< HEAD + "version": "3.12.3" +======= + "version": "3.11.2" +>>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/notebooks/devnet_datasets.ipynb b/docs/notebooks/devnet_datasets.ipynb new file mode 100644 index 0000000..e67fd3f --- /dev/null +++ b/docs/notebooks/devnet_datasets.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ea4ae65a-d555-4b54-96f9-11eed006adc2", + "metadata": {}, + "outputs": [], + "source": [ + "# %pip uninstall -y coniferest\n", + "# %pip install 'git+https://github.com/snad-space/coniferest@fix-devent-celeba'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3d9577061e9494ed", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T15:41:49.204695Z", + "start_time": "2024-03-13T15:41:49.201344Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from tqdm import tqdm\n", + "\n", + "from coniferest.aadforest import AADForest\n", + "from coniferest.datasets import Dataset, DevNetDataset\n", + "from coniferest.isoforest import IsolationForest\n", + "from coniferest.label import Label\n", + "from coniferest.pineforest import PineForest\n", + "from coniferest.session.oracle import OracleSession, create_oracle_session" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "initial_id", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T15:41:49.210919Z", + "start_time": "2024-03-13T15:41:49.206277Z" + } + }, + "outputs": [], + "source": [ + "class Compare:\n", + " models = {\n", + " 'Isolation Forest': IsolationForest,\n", + " 'AAD': AADForest,\n", + " 'Pine Forest': PineForest,\n", + " }\n", + " \n", + " def __init__(self, dataset: Dataset, *, iterations=100, n_jobs=-1):\n", + " self.model_kwargs = {\n", + " 'n_trees': 128,\n", + " 'n_jobs': n_jobs,\n", + " }\n", + " self.session_kwargs = {\n", + " 'data': dataset.data,\n", + " 'labels': dataset.labels,\n", + " 'max_iterations': iterations,\n", + " }\n", + " self.results = {}\n", + " self.steps = np.arange(1, iterations + 1)\n", + " self.total_anomaly_fraction = np.mean(dataset.labels == Label.A)\n", + "\n", + " def get_sessions(self, random_seed):\n", + " model_kwargs = self.model_kwargs | {'random_seed': random_seed}\n", + "\n", + " return {\n", + " name: create_oracle_session(model=model(**model_kwargs), **self.session_kwargs)\n", + " for name, model in self.models.items()\n", + " }\n", + "\n", + " def run(self, random_seeds):\n", + " results = defaultdict(dict)\n", + " \n", + " for random_seed in tqdm(random_seeds):\n", + " sessions = self.get_sessions(random_seed)\n", + " for name, session in sessions.items():\n", + " session.run()\n", + " anomalies = np.cumsum(np.array(list(session.known_labels.values())) == Label.A)\n", + " results[name][random_seed] = anomalies\n", + "\n", + " self.results |= results\n", + " return self\n", + " \n", + " def plot(self, dataset_name: str, savefig=False):\n", + " plt.figure(figsize=(8, 6))\n", + " plt.title(f'Dataset: {dataset_name}')\n", + "\n", + " for name, anomalies_dict in self.results.items():\n", + " anomalies = np.stack(list(anomalies_dict.values()))\n", + " q10, median, q90 = np.quantile(anomalies, [0.1, 0.5, 0.9], axis = 0)\n", + "\n", + " plt.plot(self.steps, median, alpha=0.75, label=name)\n", + " plt.fill_between(self.steps, q10, q90, alpha=0.5)\n", + "\n", + " plt.plot(self.steps, self.steps * self.total_anomaly_fraction, ls='--', color='grey', label='Theoretical radnom')\n", + "\n", + " plt.xlabel('Iteration')\n", + " plt.ylabel('Number of anomalies')\n", + " plt.grid()\n", + " plt.legend()\n", + " if savefig:\n", + " plt.savefig(f'{dataset}.pdf')\n", + " \n", + " return self" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71c337b3577915d5", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['donors', 'census', 'fraud', 'celeba', 'backdoor', 'campaign', 'thyroid']\n", + "donors\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 60%|██████████████████████████████████▏ | 12/20 [1:56:30<1:25:02, 637.84s/it]" + ] + } + ], + "source": [ + "print(DevNetDataset.avialble_datasets)\n", + "\n", + "seeds = range(20)\n", + "\n", + "for dataset in DevNetDataset.avialble_datasets:\n", + " print(dataset)\n", + " %time compare = Compare(DevNetDataset(dataset), iterations=100, n_jobs=10).run(seeds).plot(dataset, savefig=True)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "603f9b12-b5ca-470e-95ba-34e4c6571687", + "metadata": {}, + "outputs": [], + "source": [ + "%time compare = Compare(DevNetDataset(\"thyroid\"), iterations=7200, n_jobs=15).run([0]).plot(f'{dataset}_full', savefig=True)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e7fb96f-b3a4-4f33-8389-466ad23b9da6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 339831cae910403bb7f51b2ec467ea01131f0976 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 3 Jul 2025 11:38:27 -0400 Subject: [PATCH 35/40] Update notebook to use 200 realisations --- docs/notebooks/devnet_datasets.ipynb | 111 ++++++++++++++------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/docs/notebooks/devnet_datasets.ipynb b/docs/notebooks/devnet_datasets.ipynb index e67fd3f..b0a4873 100644 --- a/docs/notebooks/devnet_datasets.ipynb +++ b/docs/notebooks/devnet_datasets.ipynb @@ -6,26 +6,21 @@ "id": "ea4ae65a-d555-4b54-96f9-11eed006adc2", "metadata": {}, "outputs": [], - "source": [ - "# %pip uninstall -y coniferest\n", - "# %pip install 'git+https://github.com/snad-space/coniferest@fix-devent-celeba'" - ] + "source": "# %pip install coniferest" }, { "cell_type": "code", - "execution_count": 2, "id": "3d9577061e9494ed", "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T15:41:49.204695Z", - "start_time": "2024-03-13T15:41:49.201344Z" - }, "collapsed": false, "jupyter": { "outputs_hidden": false + }, + "ExecuteTime": { + "end_time": "2025-07-03T15:29:14.711984Z", + "start_time": "2025-07-03T15:29:13.632289Z" } }, - "outputs": [], "source": [ "from collections import defaultdict\n", "\n", @@ -39,19 +34,19 @@ "from coniferest.label import Label\n", "from coniferest.pineforest import PineForest\n", "from coniferest.session.oracle import OracleSession, create_oracle_session" - ] + ], + "outputs": [], + "execution_count": 1 }, { "cell_type": "code", - "execution_count": 3, "id": "initial_id", "metadata": { "ExecuteTime": { - "end_time": "2024-03-13T15:41:49.210919Z", - "start_time": "2024-03-13T15:41:49.206277Z" + "end_time": "2025-07-03T15:29:15.712748Z", + "start_time": "2025-07-03T15:29:15.707980Z" } }, - "outputs": [], "source": [ "class Compare:\n", " models = {\n", @@ -59,7 +54,7 @@ " 'AAD': AADForest,\n", " 'Pine Forest': PineForest,\n", " }\n", - " \n", + "\n", " def __init__(self, dataset: Dataset, *, iterations=100, n_jobs=-1):\n", " self.model_kwargs = {\n", " 'n_trees': 128,\n", @@ -84,7 +79,7 @@ "\n", " def run(self, random_seeds):\n", " results = defaultdict(dict)\n", - " \n", + "\n", " for random_seed in tqdm(random_seeds):\n", " sessions = self.get_sessions(random_seed)\n", " for name, session in sessions.items():\n", @@ -94,19 +89,20 @@ "\n", " self.results |= results\n", " return self\n", - " \n", + "\n", " def plot(self, dataset_name: str, savefig=False):\n", " plt.figure(figsize=(8, 6))\n", " plt.title(f'Dataset: {dataset_name}')\n", "\n", " for name, anomalies_dict in self.results.items():\n", " anomalies = np.stack(list(anomalies_dict.values()))\n", - " q10, median, q90 = np.quantile(anomalies, [0.1, 0.5, 0.9], axis = 0)\n", + " q5, median, q95 = np.quantile(anomalies, [0.05, 0.5, 0.95], axis=0)\n", "\n", " plt.plot(self.steps, median, alpha=0.75, label=name)\n", - " plt.fill_between(self.steps, q10, q90, alpha=0.5)\n", + " plt.fill_between(self.steps, q5, q95, alpha=0.5)\n", "\n", - " plt.plot(self.steps, self.steps * self.total_anomaly_fraction, ls='--', color='grey', label='Theoretical radnom')\n", + " plt.plot(self.steps, self.steps * self.total_anomaly_fraction, ls='--', color='grey',\n", + " label='Theoretical random')\n", "\n", " plt.xlabel('Iteration')\n", " plt.ylabel('Number of anomalies')\n", @@ -114,20 +110,35 @@ " plt.legend()\n", " if savefig:\n", " plt.savefig(f'{dataset}.pdf')\n", - " \n", + "\n", " return self" - ] + ], + "outputs": [], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": null, "id": "71c337b3577915d5", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false + }, + "ExecuteTime": { + "end_time": "2025-07-03T15:35:53.300312Z", + "start_time": "2025-07-03T15:34:16.696646Z" } }, + "source": [ + "print(DevNetDataset.avialble_datasets)\n", + "\n", + "seeds = range(200)\n", + "\n", + "for dataset in DevNetDataset.avialble_datasets:\n", + " print(dataset)\n", + " %time compare = Compare(DevNetDataset(dataset), iterations=100, n_jobs=-1).run(seeds).plot(dataset, savefig=True)\n", + " plt.show()" + ], "outputs": [ { "name": "stdout", @@ -141,39 +152,31 @@ "name": "stderr", "output_type": "stream", "text": [ - " 60%|██████████████████████████████████▏ | 12/20 [1:56:30<1:25:02, 637.84s/it]" + " 0%| | 0/200 [01:35 \u001B[39m\u001B[32m7\u001B[39m \u001B[43mget_ipython\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m.\u001B[49m\u001B[43mrun_line_magic\u001B[49m\u001B[43m(\u001B[49m\u001B[33;43m'\u001B[39;49m\u001B[33;43mtime\u001B[39;49m\u001B[33;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[33;43m'\u001B[39;49m\u001B[33;43mcompare = Compare(DevNetDataset(dataset), iterations=100, n_jobs=-1).run(seeds).plot(dataset, savefig=True)\u001B[39;49m\u001B[33;43m'\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[32m 8\u001B[39m plt.show()\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/.virtualenvs/coniferest/lib/python3.12/site-packages/IPython/core/interactiveshell.py:2488\u001B[39m, in \u001B[36mInteractiveShell.run_line_magic\u001B[39m\u001B[34m(self, magic_name, line, _stack_depth)\u001B[39m\n\u001B[32m 2486\u001B[39m kwargs[\u001B[33m'\u001B[39m\u001B[33mlocal_ns\u001B[39m\u001B[33m'\u001B[39m] = \u001B[38;5;28mself\u001B[39m.get_local_scope(stack_depth)\n\u001B[32m 2487\u001B[39m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m.builtin_trap:\n\u001B[32m-> \u001B[39m\u001B[32m2488\u001B[39m result = \u001B[43mfn\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 2490\u001B[39m \u001B[38;5;66;03m# The code below prevents the output from being displayed\u001B[39;00m\n\u001B[32m 2491\u001B[39m \u001B[38;5;66;03m# when using magics with decorator @output_can_be_silenced\u001B[39;00m\n\u001B[32m 2492\u001B[39m \u001B[38;5;66;03m# when the last Python token in the expression is a ';'.\u001B[39;00m\n\u001B[32m 2493\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mgetattr\u001B[39m(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, \u001B[38;5;28;01mFalse\u001B[39;00m):\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/.virtualenvs/coniferest/lib/python3.12/site-packages/IPython/core/magics/execution.py:1390\u001B[39m, in \u001B[36mExecutionMagics.time\u001B[39m\u001B[34m(self, line, cell, local_ns)\u001B[39m\n\u001B[32m 1388\u001B[39m st = clock2()\n\u001B[32m 1389\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m-> \u001B[39m\u001B[32m1390\u001B[39m \u001B[43mexec\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mglob\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlocal_ns\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 1391\u001B[39m out = \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[32m 1392\u001B[39m \u001B[38;5;66;03m# multi-line %%time case\u001B[39;00m\n", + "\u001B[36mFile \u001B[39m\u001B[32m:1\u001B[39m\n", + "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[2]\u001B[39m\u001B[32m, line 36\u001B[39m, in \u001B[36mCompare.run\u001B[39m\u001B[34m(self, random_seeds)\u001B[39m\n\u001B[32m 34\u001B[39m sessions = \u001B[38;5;28mself\u001B[39m.get_sessions(random_seed)\n\u001B[32m 35\u001B[39m \u001B[38;5;28;01mfor\u001B[39;00m name, session \u001B[38;5;129;01min\u001B[39;00m sessions.items():\n\u001B[32m---> \u001B[39m\u001B[32m36\u001B[39m \u001B[43msession\u001B[49m\u001B[43m.\u001B[49m\u001B[43mrun\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 37\u001B[39m anomalies = np.cumsum(np.array(\u001B[38;5;28mlist\u001B[39m(session.known_labels.values())) == Label.A)\n\u001B[32m 38\u001B[39m results[name][random_seed] = anomalies\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/session/__init__.py:158\u001B[39m, in \u001B[36mSession.run\u001B[39m\u001B[34m(self)\u001B[39m\n\u001B[32m 156\u001B[39m known_data = \u001B[38;5;28mself\u001B[39m._data[\u001B[38;5;28mlist\u001B[39m(\u001B[38;5;28mself\u001B[39m._known_labels.keys())]\n\u001B[32m 157\u001B[39m known_labels = np.fromiter(\u001B[38;5;28mself\u001B[39m._known_labels.values(), dtype=\u001B[38;5;28mint\u001B[39m, count=\u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m._known_labels))\n\u001B[32m--> \u001B[39m\u001B[32m158\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m.\u001B[49m\u001B[43mfit_known\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43m_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mknown_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mknown_labels\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 160\u001B[39m \u001B[38;5;28mself\u001B[39m._invoke_callbacks(\u001B[38;5;28mself\u001B[39m._on_refit_cb, \u001B[38;5;28mself\u001B[39m)\n\u001B[32m 162\u001B[39m \u001B[38;5;28mself\u001B[39m._scores = \u001B[38;5;28mself\u001B[39m.model.score_samples(\u001B[38;5;28mself\u001B[39m._data)\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/pineforest.py:196\u001B[39m, in \u001B[36mPineForest.fit_known\u001B[39m\u001B[34m(self, data, known_data, known_labels)\u001B[39m\n\u001B[32m 194\u001B[39m \u001B[38;5;28mself\u001B[39m._expand_trees(data, \u001B[38;5;28mself\u001B[39m.n_trees)\n\u001B[32m 195\u001B[39m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[32m--> \u001B[39m\u001B[32m196\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43m_expand_trees\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mn_trees\u001B[49m\u001B[43m \u001B[49m\u001B[43m+\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mn_spare_trees\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 197\u001B[39m \u001B[38;5;28mself\u001B[39m._contract_trees(known_data, known_labels, \u001B[38;5;28mself\u001B[39m.n_trees)\n\u001B[32m 199\u001B[39m \u001B[38;5;28mself\u001B[39m.evaluator = ConiferestEvaluator(\u001B[38;5;28mself\u001B[39m)\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/pineforest.py:101\u001B[39m, in \u001B[36mPineForest._expand_trees\u001B[39m\u001B[34m(self, data, n_trees)\u001B[39m\n\u001B[32m 99\u001B[39m n = n_trees - \u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m.trees)\n\u001B[32m 100\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m n > \u001B[32m0\u001B[39m:\n\u001B[32m--> \u001B[39m\u001B[32m101\u001B[39m \u001B[38;5;28mself\u001B[39m.trees.extend(\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mbuild_trees\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mn\u001B[49m\u001B[43m)\u001B[49m)\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/coniferest.py:117\u001B[39m, in \u001B[36mConiferest.build_trees\u001B[39m\u001B[34m(self, data, n_trees)\u001B[39m\n\u001B[32m 109\u001B[39m indices = _generate_indices(\n\u001B[32m 110\u001B[39m random_state=random_state,\n\u001B[32m 111\u001B[39m bootstrap=\u001B[38;5;28mself\u001B[39m.bootstrap_samples,\n\u001B[32m 112\u001B[39m n_population=n_population,\n\u001B[32m 113\u001B[39m n_samples=n_samples,\n\u001B[32m 114\u001B[39m )\n\u001B[32m 116\u001B[39m subsamples = data[indices, :]\n\u001B[32m--> \u001B[39m\u001B[32m117\u001B[39m tree = \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mbuild_one_tree\u001B[49m\u001B[43m(\u001B[49m\u001B[43msubsamples\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 118\u001B[39m trees.append(tree)\n\u001B[32m 120\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m trees\n", + "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/coniferest.py:122\u001B[39m, in \u001B[36mConiferest.build_one_tree\u001B[39m\u001B[34m(self, data)\u001B[39m\n\u001B[32m 118\u001B[39m trees.append(tree)\n\u001B[32m 120\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m trees\n\u001B[32m--> \u001B[39m\u001B[32m122\u001B[39m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34mbuild_one_tree\u001B[39m(\u001B[38;5;28mself\u001B[39m, data):\n\u001B[32m 123\u001B[39m \u001B[38;5;250m \u001B[39m\u001B[33;03m\"\"\"\u001B[39;00m\n\u001B[32m 124\u001B[39m \u001B[33;03m Build just one tree.\u001B[39;00m\n\u001B[32m 125\u001B[39m \n\u001B[32m (...)\u001B[39m\u001B[32m 133\u001B[39m \u001B[33;03m A tree.\u001B[39;00m\n\u001B[32m 134\u001B[39m \u001B[33;03m \"\"\"\u001B[39;00m\n\u001B[32m 135\u001B[39m \u001B[38;5;66;03m# Hollow plug\u001B[39;00m\n", + "\u001B[31mKeyboardInterrupt\u001B[39m: " ] } ], - "source": [ - "print(DevNetDataset.avialble_datasets)\n", - "\n", - "seeds = range(20)\n", - "\n", - "for dataset in DevNetDataset.avialble_datasets:\n", - " print(dataset)\n", - " %time compare = Compare(DevNetDataset(dataset), iterations=100, n_jobs=10).run(seeds).plot(dataset, savefig=True)\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "603f9b12-b5ca-470e-95ba-34e4c6571687", - "metadata": {}, - "outputs": [], - "source": [ - "%time compare = Compare(DevNetDataset(\"thyroid\"), iterations=7200, n_jobs=15).run([0]).plot(f'{dataset}_full', savefig=True)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e7fb96f-b3a4-4f33-8389-466ad23b9da6", - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 5 } ], "metadata": { From 3d388ccd0568950af258709eb91e9c1dfe1e9407 Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 10 Jul 2025 18:30:57 +0300 Subject: [PATCH 36/40] update devnet galzoo2 --- docs/notebooks/devnet.ipynb | 228 --------------------------- docs/notebooks/devnet_datasets.ipynb | 177 ++++++++++++++------- src/coniferest/datasets/__init__.py | 2 + 3 files changed, 119 insertions(+), 288 deletions(-) delete mode 100644 docs/notebooks/devnet.ipynb diff --git a/docs/notebooks/devnet.ipynb b/docs/notebooks/devnet.ipynb deleted file mode 100644 index dcf36d2..0000000 --- a/docs/notebooks/devnet.ipynb +++ /dev/null @@ -1,228 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, -<<<<<<< HEAD - "id": "ea4ae65a-d555-4b54-96f9-11eed006adc2", - "metadata": {}, - "outputs": [], - "source": [ - "# %pip uninstall -y coniferest\n", - "# %pip install 'git+https://github.com/snad-space/coniferest@fix-devent-celeba'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, -======= ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - "id": "3d9577061e9494ed", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T15:41:49.204695Z", - "start_time": "2024-03-13T15:41:49.201344Z" - }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "from coniferest.aadforest import AADForest\n", - "from coniferest.datasets import Dataset, DevNetDataset\n", - "from coniferest.isoforest import IsolationForest\n", - "from coniferest.label import Label\n", - "from coniferest.pineforest import PineForest\n", - "from coniferest.session.oracle import OracleSession, create_oracle_session" - ] - }, - { - "cell_type": "code", -<<<<<<< HEAD - "execution_count": 3, -======= - "execution_count": 2, ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - "id": "initial_id", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T15:41:49.210919Z", - "start_time": "2024-03-13T15:41:49.206277Z" - } - }, - "outputs": [], - "source": [ - "class Compare:\n", - " def __init__(self, dataset: Dataset, *, n_jobs=-1):\n", - " model_kwargs = {\n", - " 'n_trees': 128,\n", - " 'random_seed': 0,\n", - " 'n_jobs': n_jobs,\n", - " }\n", - " session_kwargs = {\n", - " 'data': dataset.data,\n", - " 'labels': dataset.labels,\n", - " 'max_iterations': 100,\n", - " }\n", - " \n", - " self.isoforest_session = create_oracle_session(\n", - " model=IsolationForest(**model_kwargs),\n", - " **session_kwargs,\n", - " )\n", - " self.aadforest_session = create_oracle_session(\n", - " model=AADForest(**model_kwargs),\n", - " **session_kwargs,\n", - " )\n", - " self.pineforest_session = create_oracle_session(\n", - " model=PineForest(**model_kwargs), #weight_ratio=4.0),\n", - " **session_kwargs,\n", - " )\n", - " \n", - " def run(self):\n", - " print(\"Running Isolation Forest\")\n", - " self.isoforest_session.run()\n", - " print(\"Running AAD Isolation Forest\")\n", - " self.aadforest_session.run()\n", - " print(\"Running Pine Forest\")\n", - " self.pineforest_session.run()\n", - " \n", - " return self\n", - " \n", -<<<<<<< HEAD - " def plot(self, dataset_name, savefig=False):\n", - " plt.figure(figsize=(8, 6))\n", - " plt.title(f'Dataset: {dataset_name}')\n", -======= - " def plot(self, title=None):\n", - " plt.figure(figsize=(8, 6))\n", - " if title is None:\n", - " title = 'AD performance curves'\n", - " plt.title(title)\n", ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - " \n", - " def performance(session):\n", - " return np.cumsum(np.array(list(session.known_labels.values())) == Label.A)\n", - "\n", -<<<<<<< HEAD - " plt.plot(performance(self.isoforest_session), label='Isolation Forest')\n", - " plt.plot(performance(self.aadforest_session), label='AAD Isolation Forest')\n", - " plt.plot(performance(self.pineforest_session), label='Pine Forest')\n", -======= - " plt.plot(performance(self.pineforest_session), label='Pine Forest')\n", - " plt.plot(performance(self.aadforest_session), label='AAD Isolation Forest')\n", - " plt.plot(performance(self.isoforest_session), label='Isolation Forest')\n", ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - " #plt.axhline(sum(self.dataset.labels == Label.A), color='grey')\n", - " plt.xlabel('number of iteration')\n", - " plt.ylabel('true anomalies detected')\n", - " plt.grid()\n", - " plt.legend()\n", -<<<<<<< HEAD - " if savefig:\n", - " plt.savefig(f'{dataset}.pdf')\n", -======= ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - " \n", - " return self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71c337b3577915d5", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "donors\n", -<<<<<<< HEAD - "Running Isolation Forest\n", - "Running AAD Isolation Forest\n", - "Running Pine Forest\n", - "CPU times: user 40min 36s, sys: 8.35 s, total: 40min 44s\n", - "Wall time: 7min 40s\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq4AAAIjCAYAAADC0ZkAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACweklEQVR4nOzdd1QUZxfH8e/SOyiCgBU79p4YjRVrYjeaWKLGEnuLNcWWGKOxx96wd01i8tp7jWLvHQVFxIIgfdmd94+NGxELqywLy/2cw9GZnZ25y1Aus888P5WiKApCCCGEEEJkcBamLkAIIYQQQojUkMZVCCGEEEJkCtK4CiGEEEKITEEaVyGEEEIIkSlI4yqEEEIIITIFaVyFEEIIIUSmII2rEEIIIYTIFKRxFUIIIYQQmYI0rkIIIYQQIlOQxlUIIbKgJUuWoFKpuH37tqlLEUKIVJPGVQiRqTxvuJ5/2NnZ4ePjQ/369ZkxYwbPnj17530fOXKE0aNH8/Tp07Qr+D3Mnj2bJUuWmLoMIYTIMKRxFUJkSmPHjmX58uXMmTOHvn37AjBgwABKlSrFuXPn3mmfR44cYcyYMdK4CiFEBmVl6gKEEOJdNGzYkIoVK+qXR4wYwZ49e/j0009p0qQJly9fxt7e3oQViveRlJSEVqvFxsbG1KUIITIQueIqhDAbtWvX5ocffuDOnTusWLFCv/7cuXN06tSJAgUKYGdnh5eXF1999RWPHz/WbzN69GiGDBkCgK+vr34owvMxoAEBAdSuXRtPT09sbW0pXrw4c+bMSVHDiRMnqF+/Pjly5MDe3h5fX1+++uqrZNtotVqmTZtGiRIlsLOzI2fOnHz99ddERETot8mfPz8XL15k//79+lpq1qypf/zmzZvcvHkzVZ+XixcvUrt2bezt7cmdOzc//fQTWq32ldvOnj2bEiVKYGtri4+PD717905xBbpmzZqULFmSS5cuUatWLRwcHMiVKxcTJ05Msb/w8HC6dOlCzpw5sbOzo0yZMixdujTZNrdv30alUjFp0iSmTZtGwYIFsbW15dKlSwD89ttvlChRAgcHB7Jly0bFihVZtWpVql67EMK8yBVXIYRZ6dChA99++y07duygW7duAOzcuZNbt27RuXNnvLy8uHjxIvPnz+fixYv8888/qFQqWrRowbVr11i9ejVTp04lR44cAHh4eAAwZ84cSpQoQZMmTbCysuKvv/6iV69eaLVaevfuDeiatHr16uHh4cHw4cNxc3Pj9u3bbNq0KVmNX3/9NUuWLKFz587069ePoKAgZs6cyenTpzl8+DDW1tZMmzaNvn374uTkxHfffQdAzpw59fuoU6cOwFtvrgoLC6NWrVokJSUxfPhwHB0dmT9//iuvRo8ePZoxY8bg7+9Pz549uXr1KnPmzCEwMFBf13MRERE0aNCAFi1a0Lp1azZs2MCwYcMoVaoUDRs2BCAuLo6aNWty48YN+vTpg6+vL+vXr6dTp048ffqU/v37Jzt+QEAA8fHxdO/eHVtbW7Jnz86CBQvo168frVq1on///sTHx3Pu3DmOHTtG27Zt3/zFIIQwP4oQQmQiAQEBCqAEBga+dhtXV1elXLly+uXY2NgU26xevVoBlAMHDujX/frrrwqgBAUFpdj+VfuoX7++UqBAAf3y77///tbaDh48qADKypUrk63ftm1bivUlSpRQatSo8cr95MuXT8mXL99rj/PcgAEDFEA5duyYfl14eLji6uqa7LWGh4crNjY2Sr169RSNRqPfdubMmQqgLF68WL+uRo0aCqAsW7ZMvy4hIUHx8vJSWrZsqV83bdo0BVBWrFihX5eYmKhUqVJFcXJyUqKiohRFUZSgoCAFUFxcXJTw8PBk9Tdt2lQpUaLEW1+nECJrkKECQgiz4+TklGx2gRevLsbHx/Po0SM+/PBDAE6dOpWqfb64j8jISB49ekSNGjW4desWkZGRALi5uQHw999/o1arX7mf9evX4+rqSt26dXn06JH+o0KFCjg5ObF3795U1XP79u1UTWW1ZcsWPvzwQypXrqxf5+HhQbt27ZJtt2vXLhITExkwYAAWFv/9aujWrRsuLi7873//S7a9k5MT7du31y/b2NhQuXJlbt26lezYXl5efPHFF/p11tbW9OvXj+joaPbv359sny1bttRf4X7Ozc2Nu3fvEhgY+NbXKoQwf9K4CiHMTnR0NM7OzvrlJ0+e0L9/f3LmzIm9vT0eHh74+voC6JvOtzl8+DD+/v44Ojri5uaGh4cH3377bbJ91KhRg5YtWzJmzBhy5MhB06ZNCQgIICEhQb+f69evExkZiaenJx4eHsk+oqOjCQ8PT6tPAwB37tyhcOHCKdYXLVo0xXavWm9jY0OBAgX0jz+XO3duVCpVsnXZsmVLNk73+bFfbIQB/Pz8kh3zuefn5EXDhg3DycmJypUrU7hwYXr37s3hw4df+VqFEOZPxrgKIczK3bt3iYyMpFChQvp1rVu35siRIwwZMoSyZcvi5OSEVqulQYMGr71J6UU3b96kTp06FCtWjClTppAnTx5sbGzYsmULU6dO1e9DpVKxYcMG/vnnH/766y+2b9/OV199xeTJk/nnn3/0x/X09GTlypWvPNbLVxwzKktLy1euVxTlnff5qnG3fn5+XL16lb///ptt27axceNGZs+ezciRIxkzZsw7H0sIkTlJ4yqEMCvLly8HoH79+oDuJqLdu3czZswYRo4cqd/u+vXrKZ778hXE5/766y8SEhLYvHkzefPm1a9/3dv6H374IR9++CHjxo1j1apVtGvXjjVr1tC1a1cKFizIrl27qFq16lun63pdPYbIly/fK1/r1atXU2z3fH2BAgX06xMTEwkKCsLf3/+djn3u3Dm0Wm2yq65XrlxJdsy3cXR0pE2bNrRp04bExERatGjBuHHjGDFiBHZ2dgbXJYTIvGSogBDCbOzZs4cff/wRX19f/RjO51cGX74SOG3atBTPd3R0BEgx/dOr9hEZGUlAQECy7SIiIlIcp2zZsgD64QKtW7dGo9Hw448/pjh+UlJSsmM7Ojq+NgwhtdNhNWrUiH/++Yfjx4/r1z18+DDFFV9/f39sbGyYMWNGstewaNEiIiMj+eSTT956rFcdOywsjLVr1+rXJSUl8dtvv+Hk5ESNGjXeuo8XpywD3dCF4sWLoyjKa8cRCyHMl1xxFUJkSlu3buXKlSskJSXx4MED9uzZw86dO8mXLx+bN2/WX4lzcXGhevXqTJw4EbVaTa5cudixYwdBQUEp9lmhQgUAvvvuOz7//HOsra1p3Lgx9erVw8bGhsaNG/P1118THR3NggUL8PT05P79+/rnL126lNmzZ9O8eXMKFizIs2fPWLBgAS4uLjRq1AjQjYP9+uuvGT9+PGfOnKFevXpYW1tz/fp11q9fz/Tp02nVqpW+njlz5vDTTz9RqFAhPD09qV27NpD66bCGDh3K8uXLadCgAf3799dPh/X8auhzHh4ejBgxgjFjxtCgQQOaNGnC1atXmT17NpUqVUp2I1Zqde/enXnz5tGpUydOnjxJ/vz52bBhA4cPH2batGnJxiG/Tr169fDy8qJq1arkzJmTy5cvM3PmTD755JNUPV8IYWZMOaWBEEIY6vl0WM8/bGxsFC8vL6Vu3brK9OnT9VMsveju3btK8+bNFTc3N8XV1VX57LPPlNDQUAVQRo0alWzbH3/8UcmVK5diYWGRbLqozZs3K6VLl1bs7OyU/PnzKxMmTFAWL16cbJtTp04pX3zxhZI3b17F1tZW8fT0VD799FPlxIkTKWqaP3++UqFCBcXe3l5xdnZWSpUqpQwdOlQJDQ3VbxMWFqZ88sknirOzswIkmxortdNhKYqinDt3TqlRo4ZiZ2en5MqVS/nxxx+VRYsWvXLqr5kzZyrFihVTrK2tlZw5cyo9e/ZUIiIikm1To0aNV05R1bFjxxQ1PXjwQOncubOSI0cOxcbGRilVqpQSEBCQbJvn02H9+uuvKfY5b948pXr16oq7u7tia2urFCxYUBkyZIgSGRmZqtcuhDAvKkV5j5H0QgghhBBCpBMZ4yqEEEIIITIFaVyFEEIIIUSmII2rEEIIIYTIFKRxFUIIIYQQmYI0rkIIIYQQIlOQxlUIIYQQQmQKZh9AoNVqCQ0NxdnZOU3iE4UQQgghRNpSFIVnz57h4+OTLCL6ZWbfuIaGhpInTx5TlyGEEEIIId4iJCSE3Llzv/Zxs29cn0cChoSE4OLiYvTjqdVqduzYoY9xFJmTnEfzIOfRPMh5NA9yHjM/Y57DqKgo8uTJ89YoZ7NvXJ8PD3BxcUm3xtXBwQEXFxf5xszE5DyaBzmP5kHOo3mQ85j5pcc5fNuwTrk5SwghhBBCZArSuAohhBBCiExBGlchhBBCCJEpmP0Y19RQFIWkpCQ0Gs1770utVmNlZUV8fHya7E+YhrmeR0tLS6ysrGRqOCGEEJlSlm9cExMTuX//PrGxsWmyP0VR8PLyIiQkRJqDTMycz6ODgwPe3t7Y2NiYuhQhhBDCIFm6cdVqtQQFBWFpaYmPjw82Njbv3aRotVqio6NxcnJ64wS6ImMzx/OoKAqJiYk8fPiQoKAgChcubDavTQghRNaQpRvXxMREtFotefLkwcHBIU32qdVqSUxMxM7OTpqCTMxcz6O9vT3W1tbcuXNH//qEEEKIzMJ8fiO/B3NqTIR4G/l6F0IIkVnJbzAhhBBCCJEpSOMqhBBCCCEyBWlcs7jbt2+jUqk4c+ZMhtiPEEIIIcTrSOOaCXXq1IlmzZplqOPnyZOH+/fvU7JkSaMee/To0ahUqhQfu3btMupx31ZT2bJlTXZ8IYQQIqvI0rMKiLRjaWmJl5dXuhyrRIkSKRrV7Nmzv9O+EhMTZT5TIYQQIpOQK64vUBSF2MSk9/6IS9QY/BxFUd657g0bNlCqVCns7e1xd3fH39+fmJgYQDet09ixY8mdOze2traULVuWbdu2vXZfGo2GLl264Ovri729PUWLFmX69On6x0ePHs3SpUv5888/9Vc79+3b98qhAvv376dy5crY2tri7e3N8OHDSUpK0j9es2ZN+vXrx9ChQ8mePTteXl6MHj36ra/XysoKLy+vZB/Pm8/z589Tu3Zt/eeie/fuREdH65/7/GrxuHHj8PHxoWjRogCEhITQunVr3NzcyJ49O82aNSM4OFj/vH379lG5cmUcHR1xc3OjatWq3LlzhyVLljBmzBjOnj2r/3wsWbLkra9BCCGEEIaTK64viFNrKD5yu0mOfWlsfRxsDD8d9+/f54svvmDixIk0b96cZ8+ecfDgQX0jPH36dCZPnsy8efMoV64cixcvpkmTJly8eJHChQun2J9WqyV37tysX78ed3d3jhw5Qvfu3fH29qZ169YMHjyYy5cvExUVRUBAAKC72hkaGppsP/fu3aNRo0Z06tSJZcuWceXKFbp164adnV2y5nTp0qUMGjSIY8eOcfToUTp16kTVqlWpW7euwZ+LmJgY6tevT5UqVQgMDCQ8PJyuXbvSp0+fZM3k7t27cXFxYefOnYAu3vX58w4ePIiVlRU//vgjrVq14ty5c1hZWdGsWTO6devG6tWrSUxM5Pjx46hUKtq0acOFCxfYtm2b/iqwq6urwbULIYQQ4u2kcc3k7t+/T1JSEi1atCBfvnwAlCpVSv/4pEmTGDZsGJ9//jkAEyZMYO/evUybNo1Zs2al2J+1tTVjxozRL/v6+nL06FHWrVtH69atcXJywt7enoSEhDcODZg9ezZ58uRh5syZqFQqihUrRmhoKMOGDWPkyJH6uURLly7NqFGjAChcuDAzZ85k9+7db2xcz58/j5OTk365ePHiHD9+nFWrVhEfH8+yZctwdHQEYObMmTRu3JgJEyaQM2dOABwdHVm4cKH+Ku2KFSvQarUsXLhQn5y2ePFismfPrr/SGhkZyaeffkrBggUB8PPz0x/fyclJfxVYCCGEEMYjjesL7K0tuTS2/nvtQ6vV8izqGc4uzgZN9G5vbflOxytTpgx16tShVKlS1K9fn3r16tGqVSuyZctGVFQUoaGhVK1aNdlzqlatytmzZ1+7z1mzZrF48WKCg4OJi4sjMTHR4JuPLl++TJUqVZJF6FatWpXo6Gju3r1L3rx5AV3j+iJvb2/Cw8PfuO+iRYuyefNm/bKtra3+mGXKlNE3rc+PqdVquXr1qr5xLVWqVLJxrWfPnuXGjRs4OzsnO058fDw3b96kQYMGdOrUifr161O3bl38/f1p3bo13t7ehnxKhBBCiExBURSWXVqGfz5/cjnlMnU5yUjj+gKVSvVOb9e/SKvVkmRjiYONVbokFFlaWrJz506OHDnCjh07+O233/juu+84duwY7u7uBu9vzZo1DB48mMmTJ1OlShWcnZ359ddfOXbsmBGq113hfZFKpUKr1b7xOTY2NhQqVOidj/liYwsQHR1NhQoVWLlypX6dVqslOjoaX19fAAICAujXrx/btm1j7dq1fP/99+zcuZMPP/zwnesQQgghMhqNVsP44+NZe3UtG65tYF3jddhb2Zu6LD25OcsMqFQqqlatypgxYzh9+jQ2Njb8/vvvuLi44OPjw+HDh5Ntf/jwYYoXL/7KfR0+fJiPPvqIXr16Ua5cOQoVKsTNmzeTbWNjY4NGo3ljTX5+fhw9ejTZTWeHDx/G2dmZ3Llzv+MrfTM/Pz/Onj2rvzHt+TEtLCz0N2G9Svny5bl+/Tqenp4UKlRI/1GgQIFk41XLlSvHiBEjOHLkCCVLlmTVqlVA6j4fQgghREaXoElg8P7BrL26FhUqvij2RYZqWkEa10zv2LFj/Pzzz5w4cYLg4GA2bdrEw4cP9WMwhwwZwoQJE1i7di1Xr15l+PDhnDlzhv79+79yf4ULF+bEiRNs376da9eu8cMPPxAYGJhsm/z583Pu3DmuXr3Ko0ePUKvVKfbTq1cvQkJC6Nu3L1euXOHPP/9k1KhRDBo0yGhXotu1a4ednR0dO3bkwoUL7N27l759+9KhQwf9MIHXPS9Hjhw0bdqUgwcPEhQUxL59+xg2bBh3794lKCiIESNGcPToUe7cucOOHTu4fv26/nOcP39+goKCOHPmDI8ePSIhIcEor08IIYQwlqjEKL7e+TW7gndhbWHNpBqTaOvX1tRlpSBDBTI5FxcXDhw4wLRp04iKiiJfvnxMnjyZhg0bAtCvXz8iIyP55ptvCA8Pp3jx4mzevPmVMwoAfP3115w+fZo2bdqgUqn44osv6NWrF1u3btVv061bN/bt20fFihWJjo5m79695M+fP9l+cuXKxZYtWxgyZAhlypQhe/bsdOnShe+//95onwsHBwe2b99O//79qVSpEg4ODrRs2ZIpU6a89XkHDhxg2LBhtGjRgmfPnpErVy4+/vhjXFxcSEhI4MqVKyxdupTHjx/j7e1N7969+frrrwFo2bIlmzZtolatWjx9+pSAgAA6depktNcphBBCpKUHMQ/osasHN57ewMnaiRm1Z1DJq5Kpy3ollfI+E4hmAlFRUbi6uhIZGYmLi0uyx+Lj4wkKCsLX1xc7O7s0OZ5WqyUqKgoXF5d0GeMqjMOcz6Mxvu4zKrVazZYtW2jUqFGK8dQi85DzaB7kPGZMt57e4utdXxMWE4aHvQdz/OdQNPurh9cZ8xy+qV97kVxxFUIIIYTIgs6En6HPnj5EJkSS3yU/8+rOw8fJx9RlvZF5XUoSQgghhBBvtS9kH912dCMyIZLSHqVZ3nB5hm9awcSNq0aj4YcfftDHixYsWJAff/wx2Z3oiqIwcuRIvL29sbe3x9/fn+vXr5uwaiGEEEKIzGvjtY3039ufeE081XNXZ0HdBbjZuZm6rFQxaeM6YcIE5syZw8yZM7l8+TITJkxg4sSJ/Pbbb/ptJk6cyIwZM5g7dy7Hjh3D0dGR+vXrEx8fb8LKhRBCCCEyF0VRmHt2LqOPjkaraGleqDnTa03HwdrB1KWlmknHuB45coSmTZvyySefALpphVavXs3x48cB3Sd42rRpfP/99zRt2hSAZcuWkTNnTv744w99jKkQQgghhHi9F4MFALqV6kbfcn2TJVxmBiZtXD/66CPmz5/PtWvXKFKkCGfPnuXQoUP66YuCgoIICwvD399f/xxXV1c++OADjh49+srGNSEhIdk8mlFRUYDuTriX5xtVq9UoioJWq31rWlNqPR/m8Hy/InMy5/Oo1WpRFAW1Wo2l5btFDWcWz7/nXzXXsMg85DyaBzmPppOgSeDbw9+y9+5eVKgYWnEobYq0ISkpyaD9GPMcpnafJm1chw8fTlRUFMWKFcPS0hKNRsO4ceNo164dAGFhYQApJo/PmTOn/rGXjR8/njFjxqRYv2PHDhwckl8Kt7KywsvLi+joaBITE9PiJek9e/YsTfcnTMMcz2NiYiJxcXEcOHDA4B9amdXOnTtNXYJIA3IezYOcx/QVp41jRcwK7mjuYIklnzl8hvMNZ7bc2PLO+zTGOYyNjU3VdiZtXNetW8fKlStZtWoVJUqU4MyZMwwYMAAfHx86duz4TvscMWIEgwYN0i9HRUWRJ08e6tWr98p5XENCQnByckqz+SwVReHZs2c4Oztnusvv4j/mfB7j4+Oxt7enevXqWWIe1507d1K3bl2ZNzITk/NoHuQ8pr8HsQ/ou7cvdzR3cLJ2Ymr1qVTIWeGd92fMc/j8HfK3MWnjOmTIEIYPH65/y79UqVLcuXOH8ePH07FjR7y8vAB48OAB3t7e+uc9ePCAsmXLvnKftra22NraplhvbW2d4pOs0WhQqVRYWFik2STzz99Wfr5fkTmZ83m0sLBApVK98nvCXGWl12rO5DyaBzmP6ePm05v02NWDsJgwPO09me0/+7XBAoYyxjlM7f5M+hs5NjY2RVNgaWmpbxp8fX3x8vJi9+7d+sejoqI4duwYVapUSddahfF16tSJZs2aZZj9CCGEEJnR6fDTfLn1S8Jiwsjvkp/ljZanWdNqaiZtXBs3bsy4ceP43//+x+3bt/n999+ZMmUKzZs3B3RXuwYMGMBPP/3E5s2bOX/+PF9++SU+Pj7SmABHjx7F0tJSPyvD66xevRpLS0t69+6d4rF9+/ahUqn0VxZdXV0pV64cQ4cO5f79+2/c7+3bt1GpVJw5c+Z9XsY7e93xp0+fzpIlS4x+/Oeftxc/qlWrZvTjvq2mP/74w6Q1CCGEMJ29wXvptqMbUYlRmSpYILVM2rj+9ttvtGrVil69euHn58fgwYP5+uuv+fHHH/XbDB06lL59+9K9e3cqVapEdHQ027ZtM/uxeamxaNEi+vbty4EDBwgNDX3jdkOHDmX16tWvnf/26tWrhIaGEhgYyLBhw9i1axclS5bk/PnzxirfaFxdXXFzc0uXYwUEBHD//n39x+bNm995X3KnrRBCiPex4doGBuwbQIImgRq5a7Cw3sJMEyyQWiZtXJ2dnZk2bRp37twhLi6Omzdv8tNPP2FjY6PfRqVSMXbsWMLCwoiPj2fXrl0UKVLEOAUpCiTGvP+HOtbw57yQFpYa0dHRrF27lp49e/LJJ5+89gpjUFAQR44cYfjw4RQpUoRNmza9cjtPT0+8vLwoUqQIn3/+OYcPH8bDw4OePXumuqaIiAjatWuHh4cH9vb2FC5cmICAAP3j58+fp3bt2tjb2+Pu7k737t2Jjo5+7f62bdtGtWrVcHNzw93dnU8//ZSbN2/qH/f19QWgXLlyqFQqatasCaQcKpCQkEC/fv3w9PTEzs6OatWqERgYqH/8+VXn3bt3U7FiRRwcHKhWrVqqEtrc3Nzw8vLSf2TPnh3QjZEdO3YsuXPnxtbWlrJly7Jt2zb9855fLV67di01atTAzs6OlStXArBw4UL8/Pyws7OjWLFizJ49W/+8xMRE+vTpg7e3N3Z2duTLl4/x48cDunmQAZo3b45KpdIvCyGEMG+KojDn7BzGHB2jDxaYVmsa9lb2pi4tzZn05qwMRx0LP7/f5XQLwO1dnvhtKNg4pnrzdevWUaxYMYoWLUr79u0ZMGAAI0aMSHEHfEBAAJ988gmurq60b9+eRYsW0bZt27fu397enh49ejBw4EDCw8Px9PR863N++OEHLl26xNatW8mRIwc3btwgLi4OgJiYGOrXr0+VKlUIDAwkPDycrl270qdPn9c23TExMQwaNIjSpUsTHR3NyJEjad68OWfOnMHCwoLjx49TuXJldu3aRYkSJZL9wfOioUOHsnHjRpYuXUq+fPmYOHEi9evX58aNG/pGE+C7775j8uTJeHh40KNHD/r06cPRo0ff+rpfZfr06UyePJl58+ZRrlw5Fi9eTJMmTbh48SKFCxfWbzd8+HAmT55MuXLl9M3ryJEjmTlzJuXKleP06dN069YNR0dHOnbsyIwZM9i8eTPr1q0jb968hISEEBISAkBgYCCenp4EBATQoEEDs5+jVQghhC5Y4OdjP7Pu2joAupfuTp+yfcxuRpznpHHNpBYtWkT79u0BaNCgAZGRkezfv19/1RF0V/2WLFmij9D9/PPP+eabbwgKCtJfrXyTYsWKAbqrg6lpXIODgylXrhwVK1YESHbFb9WqVcTHx7Ns2TIcHXUN+syZM2ncuDETJkxIMVcvQMuWLZMtL168GA8PDy5dukTJkiXx8PAAwN3dXT8DxctiYmKYM2cOS5YsoWHDhgAsWLCAnTt3smjRIoYMGaLfdty4cdSoUQPQNbuNGzcmPj4+xfy/L/riiy+SNYgrVqygWbNmTJo0iWHDhulnzJgwYQJ79+5l2rRpzJo1S7/9gAEDaNGihX551KhRTJ48Wb/O19eXS5cuMW/ePDp27EhwcDCFCxemWrVqqFQq8uXLp3/u88/H86vAQgghzFt8UjzDDgxjT8geVKj49oNv+byYeaeKSuP6ImsH3ZXP96DVaol69gwXZ2fDplEyICf46tWrHD9+nN9//x3QBSm0adOGRYsWJWtcd+7cSUxMDI0aNQIgR44c1K1bl8WLFycbR/w6z9OjUvtXW8+ePWnZsiWnTp2iXr16NGvWjI8++giAy5cvU6ZMGX3TClC1alW0Wi1Xr159ZeN6/fp1Ro4cybFjx3j06JF+tong4GBKliyZqppu3ryJWq2matWq+nXW1tZUrlyZy5cvJ9u2dOnS+v8/n34tPDz8jW+5T506NVmym7e3N1FRUYSGhiY75vPXe/bs2WTrnjf5oGuyb968SZcuXejWrZt+fVJSEq6uroBuGETdunUpWrQoDRo04NNPP6VevXpv+zQIIYQwM5EJkfTb049T4aewtrBmQvUJ1M1X19RlGZ00ri9SqQx6u/6VtFqw1uj2Y6T5PxctWkRSUhI+Pv8Na1AUBVtbW2bOnKlvchYtWsSTJ0+wt/9vjItWq+XcuXOMGTPmrY3188YutWMlGzZsyJ07d9iyZQs7d+6kTp069O7dm0mTJhn4CnUaN25Mvnz5WLBgAT4+Pmi1WkqWLJnmKWfPvTiH3PNm/W1xr15eXhQqVCjZutROogwka+Sfj/ddsGABH3zwQbLtnl/VLV++PEFBQWzdupVdu3bRunVr/P392bBhQ6qPKYQQInMLiwmj566e3Hh6A2drZ6bXnk4lr0qmLitdmNfM6llAUlISy5YtY/LkyZw5c0b/cfbsWXx8fFi9ejUAjx8/5s8//2TNmjXJtjt9+jQRERHs2LHjjceJi4tj/vz5VK9eXf8WdGp4eHjQsWNHVqxYwbRp05g/fz4Afn5+nD17lpiYGP22hw8fxsLCgqJFU84t9/jxY65evcr3339PnTp18PPzIyIiItk2z8e0ajSa19ZTsGBBbGxsOHz4sH6dWq0mMDCQ4sWLp/p1GcLFxQUfH59kxwTd633TMXPmzImPjw+3bt2iUKFCyT5eHNrh4uJCmzZtWLBgAWvXrmXjxo08efIE0DXfb/p8CCGEyNxuPr1J+y3tufH0Bp72nixpuCTLNK0gV1wznb///puIiAi6dOmiv7L6XMuWLVm0aBE9evRg+fLluLu707p16xRv9Tdq1IhFixbRoEED/brw8HDi4+N59uwZJ0+eZOLEiTx69Oi1sxC8ysiRI6lQoQIlSpQgISGBv//+Gz8/PwDatWvHqFGj6NixI6NHj+bhw4f07duXDh06vHKYQLZs2XB3d2f+/Pl4e3sTHBzM8OHDk23j6emJvb0927ZtI3fu3NjZ2aX4nDg6OtKzZ0+GDBlC9uzZyZs3LxMnTiQ2NpYuXbqk+rUZasiQIYwaNYqCBQtStmxZAgICOHPmjH7mgNcZM2YM/fr1w9XVlQYNGpCQkMCJEyeIiIhg0KBBTJkyBW9vb8qVK4eFhQXr16/Hy8tLP/1X/vz52b17N1WrVsXW1pZs2bIZ7TUKIYRIX6fDT9Nndx+iEqPwdfVlrv9cs5qjNTXkimsms2jRIvz9/VM0aKBrXE+cOMG5c+dYvHixflqkV223efNmHj16pF9XtGhRfHx8qFChAr/88gv+/v5cuHDBoKuSNjY2jBgxgtKlS1O9enUsLS1Zs2YNAA4ODmzfvp0nT55QqVIlWrVqRZ06dZg5c+Yr92VhYcGaNWs4efIkJUuWZODAgfz666/JtrGysmLGjBnMmzcPHx8fmjZt+sp9/fLLL7Rs2ZIOHTpQvnx5bty4wfbt243a1PXr149BgwbxzTffUKpUKbZt28bmzZuTzSjwKl27dmXhwoUEBARQqlQpatSowZIlS/RXXJ2dnZk4cSIVK1akUqVK3L59my1btuiHfUyePJmdO3eSJ08eypUrZ7TXJ4QQIn3tCd6jDxYo41GGZQ2WGbdpTTLOsLz3pVIUAycQzWSioqJwdXUlMjISFxeXZI/Fx8fr77BPq0ADrVZLVFQULi4uZpdxn5WY83k0xtd9RqVWq9myZQuNGjWSbPRMTM6jeZDz+O42XNvAj//8iFbRUjN3TSbWmGjcOVqj7sPKVvBhLyjXTr/amOfwTf3ai8zrN7IQQgghhJl4OVigReEWTK011bhN68NrsKguPLgAe8dBYqzxjvUOZIyrEEIIIUQGo9FqGHdsHOuvrQfSKVggJBBWtYa4J5C9IHTYBDapn64zPUjjKoQQQgiRgbwcLPDdB9/Rplgb4x702nZY1xGS4sCnPLRbD445jHvMdyCNqxBCCCFEBhGZEEnfPX05HX4aGwsbJlSfgH8+/7c/8X2cXgGb+4GigUJ1ofXS95/X3kikcRVCCCGEyADCYsLosbMHNyNv4mzjzG+1f6NCzgrGO6CiwMHJsOffNM0ybaHJDLDMuDfPSeMqhBBCCGFiNyJu0GNXDx7EPsDTwZO5/nMpnO3NUyi+F60Gtg6DwAW65WqDoM5IXYpoBiaNqxBCCCGECZ16cIo+e/rwLPEZvq6+zPOfh7eTt/EOqI6H37vDpT8BFTT4BT7sYbzjpSFpXIUQQgghTGR38G6GHRhGgiaBMh5lmFl7Jm52bsY7YNxTWNMO7hwCSxtoPg9KtjDe8dKYNK5CCCGEECaw7uo6xh0bl47BAqGwohWEXwQbZ/hiFfhWN97xjEAaVzPUqVMnnj59yh9//GHqUoQQQgjxkufBAnPOzgGgReEW/PDhD1hZGLEte3gNVrSAyBBwygntNoB3aeMdz0gkOSsT6tSpEyqVCpVKhY2NDYUKFWLs2LEkJSUBMH36dJYsWWL0Op7X8OJHtWrVjH7ct9UkDbsQQoiMKkmbxNh/xuqb1q9Lf83oKqON27SGHIfF9XRNa/aC0GVHpmxaQa64ZloNGjQgICCAhIQEtmzZQu/evbG2tmbEiBG4urqmWx0BAQE0aNBAv2xjY/PO+1Kr1ZJfLYQQwmzFJ8Uz9MBQ9obsRYWKbz/4ls+LfW7cg17dBus7ZfhggdSSK64vUBSFWHXse3/EJcUZ/BxFUQyq1dbWFi8vL/Lly0fPnj3x9/dn8+bNgO6KbLNmzfTb1qxZk379+jF06FCyZ8+Ol5cXo0ePTra/p0+f0rVrVzw8PHBxcaF27dqcPXv2rXW4ubnh5eWl/8iePTsAWq2WsWPHkjt3bmxtbSlbtizbtm3TP+/27duoVCrWrl1LjRo1sLOzY+XKlQAsXLgQPz8/7OzsKFasGLNnz9Y/LzExkT59+uDt7Y2dnR358uVj/PjxAOTPnx+A5s2bo1Kp9MtCCCGEqUUmRNJ9Z3f2huzFxsKGKTWnGL9pPbUc1rTVNa2F6kKnvzN10wpyxTWZuKQ4Plj1gUmOfaztMRys3z0P2N7ensePH7/28aVLlzJo0CCOHTvG0aNH6dSpE1WrVqVu3boAfPbZZ9jb27N161ZcXV2ZN28ederU4dq1a/pm1BDTp09n8uTJzJs3j3LlyrF48WKaNGnCxYsXKVz4v3nphg8fzuTJkylXrpy+eR05ciQzZ86kXLlynD59mm7duuHo6EjHjh2ZMWMGmzdvZt26deTNm5eQkBBCQkIACAwMxNPTU38V2NLS0uC6hRBCiLRmmmCBSbDnJ91y2XbQeHqGDhZILWlcMzlFUdi9ezfbt2+nb9++r92udOnSjBo1CoDChQszc+ZMdu/eTd26dTl06BDHjx8nPDwcW1tbACZNmsQff/zBhg0b6N69+2v3+8UXXyRrEFesWEGzZs2YNGkSw4YN4/PPdX9NTpgwgb179zJt2jRmzZql337AgAG0aPHfNByjRo1i8uTJ+nW+vr5cunSJefPm0bFjR4KDgylcuDDVqlVDpVKRL18+/XM9PDyA/64CCyGEEOklQZPA0dCjxCXFJVufqElkxukZhMeGS7BAGpDG9QX2VvYca3vsvfah1Wp59uwZzs7OWFikfiSGodNf/P333zg5OaFWq9FqtbRt2zbF2/8vKl06+SBsb29vwsPDATh79izR0dG4u7sn2yYuLo6bN2++sY6pU6fi7/9fhrK3tzdRUVGEhoZStWrVZNtWrVo1xfCDihUr6v8fExPDzZs36dKlC926ddOvT0pK0o/b7dSpE3Xr1qVo0aI0aNCATz/9lHr16r2xRiGEEMKYIuIj6L27N+cfnX/tNgVcCzDXf67xgwU2dYPLmwEVNJwAH3xtvOOZgDSuL1CpVO/1dj3oGtckqyQcrB0MalwNVatWLebMmYONjQ0+Pj5YWb35VL5805NKpUKr1QIQHR2Nt7c3+/btS/E8Nze3N+7Xy8uLQoUKJVsXFRX19hfwL0dHR/3/o6OjAViwYAEffJB8yMbzq7rly5cnKCiIrVu3smvXLlq3bo2/vz8bNmxI9TGFEEKItHL32V167urJ7ajbONs4Uzx78RTb5HbOzcAKA3G1NeLN03FPdeNZ7xzOlMECqSWNaybl6OiYomF8V+XLlycsLAwrK6s0uaHJxcUFHx8fDh8+TI0aNfTrDx8+TOXKlV/7vJw5c+Lj48OtW7do167dG/ffpk0b2rRpQ6tWrWjQoAFPnjwhe/bsWFtbo9Fo3vs1CCGEEG9z9clVeuzqwaO4R3g7ejO37lwKuBZI/0JeDBawdYHPV2a6YIHUksZV4O/vT5UqVWjWrBkTJ06kSJEihIaG8r///Y/mzZsnezs/tYYMGcKoUaMoWLAgZcuWJSAggDNnzuhnDnidMWPG0K9fP1xdXWnQoAEJCQmcOHGCiIgIBg0axJQpU/D29qZcuXJYWFiwfv16vLy89FeG8+fPz+7du6latSq2trZky5btXT4lQgghxBsdv3+c/nv7E62OpnC2wsz1n4ung2f6F/LwKqxo+W+wgBe03wBepdK/jnQijatApVKxZcsWvvvuOzp37szDhw/x8vKievXq5MyZ85322a9fPyIjI/nmm28IDw+nePHibN68OdmMAq/StWtXHBwc+PXXXxkyZAiOjo6UKlWKAQMGAODs7MzEiRO5fv06lpaWVKpUiS1btuiHZUyePJlBgwaxYMECcuXKxe3bt9+pfiGEEOJ1tt3exrcHv0WtVVMxZ0Wm156Oi41L+hcSchxWtYa4CHAvBO03QbZ8b39eJqZSDJ1ANJOJiorC1dWVyMhIXFySf1HFx8cTFBSEr68vdnZ2aXI8rVZLVFQULi4uRh3jKozLnM+jMb7uMyq1Ws2WLVto1KiRhFtkYnIezYO5nMeVl1cy4fgEFBTq5qvL+I/HY2tpm/6FvBgskKsitF0Hju5vfdr7MOY5fFO/9iK54iqEEEII8RaKojD91HQWXVgEwOdFP2d45eFYWphgzvBTy+CvAaBooHA9+GwJ2Di+7VlmQRpXIYQQQog3UGvVjD4yms03dQmV/cr1o2uprqjSe25URYEDk2Cv+QULpJY0rkIIIYQQrxGrjuWb/d9w6N4hLFWWjKoyiuaFm6d/IVoNbB0KgQt1yx8Phtrfm02wQGpJ4yqEEEII8QpP4p/QZ3cfzj86j52lHZNrTqZ6bhNMM6WOh01d4fJf6IIFJsIHr0+1NGfSuKIbtyJEViFf70II8XYvBgu42royq84syniUSf9CXg4WaLEASjRL/zoyiCzduD6/Iy42NhZ7e8MiV4XIrGJjY4GUaWpCCCF0rjy5Qs9dPTNIsEBLCL/0b7DAKvD9OP3ryECydONqaWmJm5sb4eHhADg4OLz3QGutVktiYiLx8fFmN41SVmKO51FRFGJjYwkPD8fNzU0foyuEEOI/x+4fo//e/sSoY0wfLLC8BUTd/TdYYCN4lUz/OjKYLN24Anh5eQHom9f3pSgKcXFx2Nvbp//dhiLNmPN5dHNz03/dCyGE+M+2oG2MODSCJG2SaYMFgo/pggXin4J7YeiwCdzypn8dGVCWb1xVKhXe3t54enqiVqvfe39qtZoDBw5QvXp1eSs2EzPX82htbS1XWoUQ4hUyTrDAVljfOV2DBTKTLN+4PmdpaZkmv9AtLS1JSkrCzs7OrBqerEbOoxBCmJ/D9w4z+uhoHsU9SvFYkjYJMHGwwMml8PcAULRZLlggtaRxFUIIIYTZ++vmX4w8PJIkJemVj1uprOhVtpcJgwV+hb3jdMtZMFggtaRxFUIIIYTZUhSFJReXMOXkFAAa+jZkUIVBqEjenDpYO+Bs45z+BWo1sGUwnFisW86iwQKpJY2rEEIIIcySVtEy6cQkll9aDsCXxb/km4rfYKHKILPFqONhYxe48jdZPVggtaRxFUIIIYTZSdQk8v2h79l6eysAgysOpmOJjiau6gVxEbC6LQQfkWABA0jjKoQQQgizEp0YzYB9Azh2/xhWKit+rPYjnxb41NRl/SfyHqxsJcEC70AaVyGEEEKYjUdxj+i5qydXnlzB3sqeaTWn8VGuj0xd1n/Cr+jSsCRY4J1I4yqEEEIIs3An6g5f7/yae9H3yG6Xndn+synhXsLUZf0n+B9Y1UaCBd6DNK5CCCGEyPQuPLpAr129iEiIII9zHub6zyWvSwZqCq/8DzZ8BUnxEizwHqRxFUIIIUSmdujeIQbtG0RcUhx+2f2Y7T+bHPY5TF3Wf04ugb8H/hssUB8+C5BggXeUQeaDEEIIIYQw3F83/6Lv7r7EJcVRxbsKAQ0CMk7TqiiwbwL81V/XtJZrr7sRS5rWdyZXXIUQQgiR6SiKQsDFAKaenApAI99G/FT1J6wzStqUVgP/+wZOBuiWJVggTUjjKoQQQohMRato+TXwV1ZcXgFAx+IdGVRxUAYKFoiDjV3/CxZo9CtU7mbqqsyCNK5CCCGEyDQSNYl8d+g7tt3eBmTUYIEvIPioLlig5UIo3tTUVZkNaVyFEEIIkSlEJ0YzYO8AjoUdw8rCip+q/sQnBT4xdVn/ibynm6P14WWwdYUvVkH+aqauyqxI4yqEEEKIDEOj1bD6ymquRVxL8di5h+e4GXkTBysHptaaykc+GTRYwNlbFyyQMwPNIWsmpHEVQgghRIaQoElg+IHh7Are9dptsttlZ47/HIq7F0/Hyt7ixWCBHEV0TasECxiFNK5CCCGEMLmoxCj67enHyQcnsbawpmOJjjhaJ582ytrCmvr56+Pl6GWiKl/hxWCB3JV0wQIO2U1dldmSxlUIIYQQJvUg5gE9dvXgxtMbOFk7MaP2DCp5VTJ1WW/3YrBAkQbQKgBsHExdlVmTxlUIIYQQJnPr6S2+3vU1YTFheNh7MMd/DkWzFzV1WW+mKLB/Iuz7WbdcrgN8Og0spa0yNvkMCyGEEMIkzoSfoc+ePkQmRJLfJT9z684ll1MuU5f1Zi8HC1QfArW+k2CBdCKNqxBCCCHS3b6QfQzZP4R4TTylc5RmZp2ZZLPLZuqy3kyCBUxOGlchhBBCpKuN1zYy9p+xaBUt1XNX59fqv+JgncHHhsY+0QULhPwDlrb/Bgs0MXVVWY40rkIIIYRIF4qisPDCQmafmw1A80LNGVllJFYWGbwdibz7b7DAlX+DBVZD/qqmripLyuBfKUIIIYQwBxqthr/i/uL4ueMAdCvVjb7l+qLK6GNDwy//GyxwT4IFMgBpXIUQQgiRZs6EnyH4WXCK9btu7+J44nFUqBheeTht/dqaoDoDBf8Dq1pDfKQEC2QQ0rgKIYQQ4r0pisLMMzOZf27+a7exxJLx1cbTsGDDdKzsHV3+GzZ2+TdYoDK0XSvBAhmANK5CCCGEeC9J2iTGHh3L7zd+B6CSVyVsLG2SbWNrYUuBiAL45/U3RYmGOREA/xskwQIZkDSuQgghhHhncUlxDNk/hP1392OhsuCHD3+gVZFWKbZTq9Vs2bLFBBUaQFFg/wTYN163LMECGY6cCSGEEEK8k6fxT+m9pzfnHp7D1tKWidUnUjtvbVOX9W5SBAsMhVrfSrBABiONqxBCCCEMFhodytc7v+Z21G1cbFyYWWcm5TzLmbqsd/NysMAnk6BSV1NXJV5BGlchhBBCGORaxDV67uxJeFw4OR1yMq/uPAq6FTR1We9GggUyFWlchRBCCJFqgWGB9N/Tn2fqZxRyK8Qc/zl4OXqZuqx382KwgJ0rfLEG8n1k6qrEG0jjKoQQQohU2XF7B8MPDketVVPeszwzas/A1dbV1GW9m2TBAj7/BgsUN3VV4i2kcRVCCCHEW62+sprxx8ajoFAnbx1++fgX7KzsTF3Wu7lzFFa3+TdYoOi/wQJ5TF2VSAVpXIUQQgjxWoqi8Nvp31hwfgEArYu05tsPvsXSwtLElb0jCRbI1KRxFUIIIcQrvRws0Ltsb74u/TWqzDpF1InFuimvFC0UbQQtF0mwQCYjjasQQgghUohLimPw/sEcuHsAC5UFIz8cScsiLU1d1rtRFNj3C+z/Rbdc/kv4ZKoEC2RCcsaEEEIIkUxEfAR99vTRBwv8Wv1XauWtZeqy3o0mSRffemqpbrnGMKg5QoIFMilpXIUQQgih93KwwKw6syjrWdbUZb0bdRxs+AqubkGCBcyDNK5CCCGEAODqk6v03NWTh3EP8XL0Yp7/PAq4FTB1We8m9gms/hxCjkmwgBmRxlUIIYQQBIYF0m9PP6LV0RRyK8Rc/7nkdMxp6rLeTeRdWN4CHl2VYAEzI42rEEIIkcVtv72dEQdHmEewwINLumCBZ6ESLGCGpHEVQgghsrBVl1fxy/FfUFDwz+vPL9V/wdbS1tRlvZs7R3TDA54HC3TYBK65TV2VSEPSuAohhBBZkKIozDg9g4XnFwLQpmgbRlQekYmDBf6CDV1AkwB5PtAND5BgAbMjjasQQgiRxai1asYcGcOfN/8EoE/ZPnQv3T3zBgsELoItg/8LFmi1GKztTV2VMAJpXIUQQogsJFYdy+D9gzl476CZBAuMh/0TdMvlO8InUyRYwIzJmRVCCCGyiIj4CPrs7sO5R7pggUk1JlEzT01Tl/VuNEnwv4FwapluWYIFsgRpXIUQQogs4F70PXrs7MHtqNu42roys/bMzBsskBirCxa4thVUFtBoElTqYuqqRDqQxlUIIYQwc1efXKXHrh48intkHsECq9rA3eO6YIFWi8CvsamrEulEGlchhBDCjB2/f5z+e/ubR7DA0xDdHK36YIG1kK+KqasS6UgaVyGEECKTC4kKYeqpqTyOe5zisfOPzqPWqqmQswIzas/AxcbFBBWmgQcX/w0WuC/BAlmYNK5CCCFEJnbx8UV67erFk/gnr90m0wcL3D4Mq7+AhEjwKKZrWiVYIEuSxlUIIYTIpI6EHmHg3oHEJsXil92PrqW6ppiL1dXGlQo5K2TeYIFLm2Fj13+DBT6EL1ZLsEAWZmHqAu7du0f79u1xd3fH3t6eUqVKceLECf3jiqIwcuRIvL29sbe3x9/fn+vXr5uwYiGEEML0/r71N7139SY2KZYPvD9gcf3F1Mtfj7r56ib7qOxdOfM2rYELYd2Xuqa16Cfw5R/StGZxJm1cIyIiqFq1KtbW1mzdupVLly4xefJksmXLpt9m4sSJzJgxg7lz53Ls2DEcHR2pX78+8fHxJqxcCCGEMJ2lF5cy4uAIkpQkGvo2ZE6dOTjZOJm6rLSjKLD3Z/jfN4ACFTpB62WShiVMO1RgwoQJ5MmTh4CAAP06X19f/f8VRWHatGl8//33NG3aFIBly5aRM2dO/vjjDz7//PMU+0xISCAhIUG/HBUVBYBarUatVhvrpeg9P0Z6HEsYj5xH8yDn0TzIefyPVtEy7fQ0VlxZAUDbom0ZVH4QaHUxrhlZqs+jNgnLrYOxOKN7jZqPh6L9eAhoFcjgr9HcGfN7MbX7VCmKoqT50VOpePHi1K9fn7t377J//35y5cpFr1696NatGwC3bt2iYMGCnD59mrJly+qfV6NGDcqWLcv06dNT7HP06NGMGTMmxfpVq1bh4OBgtNcihBBCGFOSksSm2E2cU58DoIFdA6raVk0xpjUzs9QmUOH2bLwjT6Og4myejtzJUdvUZYl0EBsbS9u2bYmMjMTF5fUzX5i0cbWzswNg0KBBfPbZZwQGBtK/f3/mzp1Lx44dOXLkCFWrViU0NBRvb2/981q3bo1KpWLt2rUp9vmqK6558uTh0aNHb/xEpBW1Ws3OnTupW7cu1tbWRj+eMA45j+ZBzqN5kPMIMeoYhhwcwj9h/2ClsmLkhyP51PdTU5dlkLeex7gILNe1w+LucRQrOzTN5qMUbZT+hYrXMub3YlRUFDly5Hhr42rSoQJarZaKFSvy888/A1CuXDkuXLigb1zfha2tLba2Kaf7sLa2TtcfeOl9PGEcch7Ng5xH85BVz+OjuEf02t2Ly08uY29lz5SaU6iWq5qpy3pnrzyPLwULqL5Yi5UEC2RYxvheTO3+THpzlre3N8WLJ5882M/Pj+DgYAC8vLwAePDgQbJtHjx4oH9MCCGEMFfBUcF02NKBy08uk90uO4vrL87UTesrPbgEi+rpmlaXXPDVdknDEq9l0sa1atWqXL16Ndm6a9eukS9fPkB3o5aXlxe7d+/WPx4VFcWxY8eoUkW+qIUQQpivi48u0mFrB+5G3yWXUy6WNVxGyRwlTV1W2rp9GBY3gGehumCBLjvA08/UVYkMzKRDBQYOHMhHH33Ezz//TOvWrTl+/Djz589n/vz5AKhUKgYMGMBPP/1E4cKF8fX15YcffsDHx4dmzZqZsnQhhBDCaI7cO8KAfQOIS4rDL7sfs/1nk8M+h6nLSlsvBgvkraILFrDP9vbniSzNpI1rpUqV+P333xkxYgRjx47F19eXadOm0a5dO/02Q4cOJSYmhu7du/P06VOqVavGtm3b9Dd2CSGEEObkr5t/MfLwSJKUJD7w/oBpNaeZ1xytoAsW+N9gQIFin0LLhTJHq0gVk0e+fvrpp3z66evvjFSpVIwdO5axY8emY1VCCCFE+lIUhaUXlzL55GQAGvo2ZFzVcVhbmtENaYqCxb7xcFj3GqnQGT6ZDJk12UukO5M3rkIIIURWp1W0TDoxieWXlgPQ3q89QyoNwUJl8mT2tKNNomzIYiwf79ct1/wWagwFM5qHVhifNK5CCCFEOjkTfoYbT2+kWH8k9Ag77+wEYFCFQXQq0cmsggVIjMVyfUfyPd6PorJA9elUXYyrEAaSxlUIIYQwMkVRWHh+ITNOz3jtNlYqK8ZWHUvjgo3TsbJ0EPsEVrXG4m4gGpU1SstFWJVsauqqRCYljasQQghhRBqthl+O/8Kaq2sA+MDrA+xfuhHJxsKGz4t9TiWvSqYo0XieBsPyFvD4OoqdG0fy9OFDScMS70EaVyGEEMJIEjQJjDg4gp13dqJCxbDKw2jn1+7tTzQHYRd0aVjRYeCSi6TP1/Ek8KapqxKZnDSuQgghhBFEJUbRf09/Tjw4gbWFNT9//DMN8jcwdVnp4/YhWN0WEiLBww/abwQHT0AaV/F+pHEVQggh3pFW0XL+0Xli1DHJ1mu0Gqaemsr1iOs4WjsyvdZ0PvD+wERVprOLf8CmbqBJhLwfwRerdMECarWpKxNmIFWN67lz51K9w9KlS79zMUIIIURmEZ8Uz5ADQ9gXsu+12+Swz8Ec/zkUy14s3eoyqeMLYMsQJFhAGEuqGteyZcuiUqlQFOWt03NoNJo0KUwIIYTIqCITIumzuw9nHp7BxsIGX1ffFNt4OXoxvPJwcjvnNkGF6UxRYM+PcPDfYIGKX0GjSRIsINJcqhrXoKAg/f9Pnz7N4MGDGTJkCFWqVAHg6NGjTJ48mYkTJxqnSiGEECKDCIsJo8fOHtyMvImzjTMza8+kfM7ypi7LdDRJ8Fd/OLNCt1zrO6g+RIIFhFGkqnHNly+f/v+fffYZM2bMoFGj/6azKF26NHny5OGHH36gWbNmaV6kEEIIkRFcj7hOj109CI8Nx9PBk7n+cymcrbCpyzKdxBhY3xmubweVBXw6DSp0NHVVwowZfHPW+fPn8fVN+ZaIr68vly5dSpOihBBCiIzm5IOT9N3Tl2eJzyjoWpC5defi5ehl6rJMJ+YxrGoN906AlR20CoBiMkerMC6DQ5D9/PwYP348iYmJ+nWJiYmMHz8ePz+/NC1OCCGEyAh239lN9x3deZb4jHKe5VjacGnWbloj7sDi+rqm1c4NvtwsTatIFwZfcZ07dy6NGzcmd+7c+hkEzp07h0ql4q+//krzAoUQQghTWnd1HeOOjUOraKmVpxYTq0/EzsrO1GWZTrJggdy6OVo9s8isCcLkDG5cK1euzK1bt1i5ciVXrlwBoE2bNrRt2xZHR8c0L1AIIYQwBUVRmH12NnPPzgWgVZFWfPfBd1hZZOEp0IMOwpq2kBAFnsWh3QZwzWXqqkQW8k7ffY6OjnTv3j2taxFCCCEyhCRtEj/98xMbr28EoFeZXvQo0+OtU0KatdcFCwiRjgwe4wqwfPlyqlWrho+PD3fu3AFg6tSp/Pnnn2lanBBCCJHe4pLiGLhvIBuvb8RCZcEPH/5Az7I9s3bTemw+rO+ka1qLfQodNknTKkzC4MZ1zpw5DBo0iIYNGxIREaEPHMiWLRvTpk1L6/qEEEKIdBOZEEn3Hd3ZF7IPW0tbptScQuuirU1dlukoCuweC1v/TcOq2AVaL5M0LGEyBjeuv/32GwsWLOC7777Dyuq/kQYVK1bk/PnzaVqcEEIIkV7uR9/ny61fcubhGZxtnJlfdz518tYxdVmmo1HDn33+S8Oq9T18MlnSsIRJGTzGNSgoiHLlyqVYb2trS0xMTJoUJYQQQqSnaxHX6LmzJ+Fx4eR0yMlc/7kUylbI1GWZTmKMbmjA9R0SLCAyFIOvuPr6+nLmzJkU67dt2ybzuAohhMh0ToSdoNPWToTHhVPQtSArGq3I2k1rzGNY2kTXtFrZw+erpGkVGYbBV1wHDRpE7969iY+PR1EUjh8/zurVqxk/fjwLFy40Ro1CCCGEUey8s5PhB4aTqE2knGc5fqv9G662rqYuy3Qi7ujmaH18XXfzVdt1kKeyqasSQs/gxrVr167Y29vz/fffExsbS9u2bfHx8WH69Ol8/vnnxqhRCCGEeGdXnlxh9JHRhMWEpXjsSfwTFBQJFgAIOw8rWumCBVzz6IIFPIqauiohknmneVzbtWtHu3btiI2NJTo6Gk9Pz7SuSwghhHhv/9z/hwF7BxCjfv09GK2LtGbEByOyeLDAAVjT7r9ggfYbwcXH1FUJkYLB36W1a9dm06ZNuLm54eDggIODAwBRUVE0a9aMPXv2pHmRQgghhKG2Bm3l20PfkqRNopJXJYZUHIKFKvmtHU42TuRyyuLJTxd/h03ddXO05quqG9Nq72bqqoR4JYMb13379pGYmJhifXx8PAcPHkyTooQQQoj3sfzSciYGTgSgXr56jP94PDaWNiauKgM6Ng+2DgMU8GsMLRaCdRYeLiEyvFQ3rufOndP//9KlS4SF/TdWSKPRsG3bNnLlyuJ/tQohhDApraJl2qlpBFwIAKBtsbYMqzwsxZXWLO95sMChKbrlSl2h4USZo1VkeKluXMuWLYtKpUKlUlG7du0Uj9vb2/Pbb7+laXFCCCFEaqm1akYdHsVft/4CoH/5/nQp2SVrR7W+ikYNm/vB2VW65drfw8eDQT5PIhNIdeMaFBSEoigUKFCA48eP4+HhoX/MxsYGT09PLC3lLzUhhBDpL1Ydy6B9gzgcehhLlSWjPxpNs0LNTF1WxpMYA+s6wo2doLKExtOg/JemrkqIVEt145ovXz4AtFqt0YoRQgghDPUk/gm9d/XmwuML2FvZM6nGJKrnrm7qsjKemMew6jO4d1IXLPDZEijawNRVCWEQgwf9jB8/nsWLF6dYv3jxYiZMmJAmRQkhhBCpEfIshA5bOnDh8QXcbN1YWG+hNK2vEnEbFtfTNa322aDjZmlaRaZkcOM6b948ihUrlmJ9iRIlmDt3bpoUJYQQQrzN5ceX6bClA8HPgsnllIvlDZdT2qO0qcvKeMLOw6J68PiGLljgq+2ShiUyLYOnwwoLC8Pb2zvFeg8PD+7fv58mRQkhhBBvcjT0KAP3DSRGHUPRbEWZ4z8HDwePtz8xq0kWLFAC2m+QYAGRqRl8xTVPnjwcPnw4xfrDhw/j4yPfDEIIIYxra9BWeu3uRYw6hspelQloECBN66tc2AQrWuqa1nxVofMWaVpFpmfwFddu3boxYMAA1Gq1flqs3bt3M3ToUL755ps0L1AIIYR47sVggfr56/NztZ8lWOBVkgULNIEWCyRYQJgFgxvXIUOG8PjxY3r16qVP0LKzs2PYsGGMGDEizQsUQgghtIqWaaensezyMgDa+bVjaKWhEizwMkWB3WPg0FTdsgQLCDNjcOOqUqmYMGECP/zwA5cvX8be3p7ChQtja2trjPqEEEJkcWqtmk2xmzhz+QwgwQKvJcECIgswuHF9LiwsjCdPnlC9enVsbW1RFEV+iAghhEhTsepYBuwfwBn1GSxVloz5aAxNCzU1dVmmdelPuHsi5frQ03D74L/BAtOhfIf0r00IIzO4cX38+DGtW7dm7969qFQqrl+/ToECBejSpQvZsmVj8uTJxqhTCCFEFvM47jG9d/fm4uOLWGPNlBpTqJmvpqnLMh2tFnZ8D//Mev02VvbQeikUqZ9+dQmRjgxuXAcOHIi1tTXBwcH4+fnp17dp04ZBgwZJ4yqEEOK9hTwLocfOHgQ/C8bN1o3PrT+nqk9VU5dlOkmJ8EdPuLBBt1y2PThkS76NhRWUbAVeJdO/PiHSicGN644dO9i+fTu5c+dOtr5w4cLcuXMnzQoTQgiRNV16fIleu3rxOP4xuZxyMbPmTC4eumjqskwn4RmsbQ+39uma06azoUwbU1clhEkYfDtmTEwMDg4OKdY/efJEbtASQgjxXo6GHqXzts48jn9M0WxFWd5wOflc8pm6LNOJDocln+iaVmtHaLtOmlaRpRncuH788ccsW7ZMv6xSqdBqtUycOJFatWqlaXFCCCGyji23ttBrdy9ik2IlWADg8U1YVBfunwWHHNDpbyhUx9RVCWFSBg8VmDhxInXq1OHEiRMkJiYydOhQLl68yJMnT16ZqCWEEEK8zbKLy/j1xK+ABAsAcO8UrPwMYh9BtvzQfhO4FzR1VUKYnMGNa8mSJbl27RozZ87E2dmZ6OhoWrRoQe/evfH29jZGjUIIIcyUVtEy9eRUllxcAkiwAAA3dsPaDqCOAe8y0G4DOHmauiohMgSDG9fg4GDy5MnDd99998rH8ubNmyaFCSGEMG9qjZqRR0by962/ARhQfgBflfwqa88JfnYt/NkLtElQoCa0WQG2zqauSogMw+A/aX19fXn48GGK9Y8fP8bX1zdNihJCCGHeYtQx9NnTh79v/Y2lypKfqv5El1JZOA1LUeDwDPi9u65pLfUZtF0vTasQLzH4iuvrErKio6Oxs7NLk6KEEEJkflpFy9mHZ4lKiEqxfu65uVx6fAl7K3sm15jMx7k/NlGVGcDLwQJV+kDdH8EiCw+XEOI1Ut24Dho0CNDNIvDDDz8kmxJLo9Fw7NgxypYtm+YFCiGEyHwSNYl8e+hbtt/e/tptstlmY1adWZTyKJWOlWUwSQnwR6//ggXq/QQf9TVtTUJkYKluXE+fPg3orrieP38eG5v/7va0sbGhTJkyDB48OO0rFEIIkak8S3zGgL0DOB52HCsLK4plK5Zimxz2Ofim4jfkd82f/gVmFPFRumCBoP26YIFmc6B0a1NXJUSGlurGde/evQB07tyZ6dOn4+LiYrSihBBCZE4PYx/Sa3cvrjy5goOVA9NqTaOKTxVTl5XxPHsAK1tB2DmwcYLWy2SOViFSweAxrgEBAQDcuHGDmzdvUr16dezt7V879lUIIUTWcDvyNj129eBe9D3c7dyZ7T+b4u7FTV1WxvP4JixvDk/v6IIF2m8An3KmrkqITMHgkd9PnjyhTp06FClShEaNGnH//n0AunTpwjfffJPmBQohhMj4zj08x5dbv+Re9D3yOudleaPl0rS+yr2TsKiermnN5gtddkjTKoQBDG5cBwwYgLW1NcHBwclu0GrTpg3btm1L0+KEEEJkfAfvHqTrjq5EJERQwr0EyxouI49zHlOXlfHc2AVLGuvSsLzL6JpWScMSwiAGDxXYsWMH27dvJ3fu3MnWFy5cmDt37qRZYUIIITK+P2/8yagjo9AoGqr6VGVKzSk4WDu8/YlZzdk18Gfvf4MFakGb5TJHqxDvwODGNSYmJtmV1ueePHmCra1tmhQlhBAiY1MUhUUXFjH91HQAGhdozJiqY7C2sDZxZRmMosCRGbBzpG651GfQdDZY2bz5eUKIVzJ4qMDHH3/MsmXL9MsqlQqtVsvEiROpVatWmhYnhBAi49EqWiYETtA3rZ1LdmZctXHStL5Mq4Xt3/3XtFbpA83nS9MqxHsw+IrrxIkTqVOnDidOnCAxMZGhQ4dy8eJFnjx5wuHDh41RoxBCiAzi5WCBoZWG0qF4BxNXlQElJcAfPeHCRt2yBAsIkSYMblxLlizJtWvXmDlzJs7OzkRHR9OiRQt69+6Nt7e3MWoUQgiRATxLfEb/vf0JDAvEysKKn6v9TEPfhqYuK+NJFixg/W+wwGemrkoIs2Bw4wrg6urKd999l9a1CCGEyKAexj6k566eXI24iqO1I9NrTecD7w9MXVbG83KwQJvlULC2qasSwmykqnE9d+5cqndYunTpdy5GCCFExhMUGUSPnT0IjQnF3c6dOf5z8HP3M3VZGc+LwQKOHtBuvczRKkQaS1XjWrZsWVQqVYp0LEVRAJKt02g0aVyiEEIIUzn38By9d/fmacJT8rnkY67/XHI75377E7Oaeydh5WcQ+1gXLNBhE2QvYOqqhDA7qZpVICgoiFu3bhEUFMTGjRvx9fVl9uzZnDlzhjNnzjB79mwKFizIxo0bjV2vEEKIdHLg7gG67ujK04SnlHQvybKGy6RpfZXru2DJp7qm1busLlhAmlYhjCJVV1zz5cun//9nn33GjBkzaNSokX5d6dKlyZMnDz/88APNmjVL8yKFEEKkrz9u/MHoI6N1wQK5qjKlhgQLvNKLwQIFa0Pr5WDrZOqqhDBbBt+cdf78eXx9fVOs9/X15dKlS2lSlBBCCNN4OVigScEmjP5otMzR+jJFgcPTYdco3XKp1tB0lszRKoSRGRxA4Ofnx/jx40lMTNSvS0xMZPz48fj5yWB9IYTIrDRaDb8c/0XftH5V8it+qvqTNK0v02ph24j/mtaP+kLzedK0CpEODL7iOnfuXBo3bkzu3Ln1MwicO3cOlUrFX3/9leYFCiGEML4ETQLfHvyWHXd2oELF0EpDaV+8vanLyniSEuD3HnBxk2653jj4qI9paxIiCzG4ca1cuTK3bt1i5cqVXLlyBYA2bdrQtm1bHB0d07xAIYQQxvVysMD4auNp4NvA1GVlPPFRsLYdBB34N1hgNpRubeqqhDCKqHg1zrZWyWaOygjeKYDA0dGR7t27p3UtQggh0ll4bDg9d/XkWsQ1CRZ4k2dh/wYLnJdgAWH2bj6M5stFx+lQJR89ahQ0dTnJGDzGVQghhHm4FXmL9lvacy3iGjnsc7CkwRJpWl/l0Q1YVFfXtDp6QKf/SdMqzNbp4AhazTnCvadxrAsMIS4xY83P/05XXIUQQmRuZx+epc/uPhIs8DZ3T8IqCRYQWcOeKw/otfIU8WotZXK7srhTJextLE1dVjLSuAohRBazP2Q/g/cPJl4TT0n3kszyn0V2u+ymLivjub4T1n0J6lhdsEC7DeDkYeqqhDCKdSdCGLHpPBqtQs2iHsxqWx5H24zXJma8ioQQQhjN79d/Z8zRMRIs8DZnVsPmPhIsIMyeoijM3neTX7dfBaBl+dz80rIU1pYZczSpwY1rSEgIKpWK3Ll1bykdP36cVatWUbx4cblhSwghMihFUVhwfgG/nf4NkGABQDcM4NgcUMclX58UDzd26f5fug00mSlztIpMb/+1h6wLDEGt0SZbHxWv5p9bTwDoWbMgQ+sXzXAzCbzI4Ma1bdu2dO/enQ4dOhAWFkbdunUpUaIEK1euJCwsjJEjRxqjTiGEEO/oebDAmqtrAOhSsgv9y/fP0L+cjO7KFtjQWdekvk7V/lBnNFhkzCtPQqTWymN3+OGPC2iVVz+uUsHIT4vTuWrKZNSMxuDG9cKFC1SuXBmAdevWUbJkSQ4fPsyOHTvo0aOHNK5CCJGBJGgSGHFwBDvv7ESFimGVh9HOr52pyzKtk0vh7wGgaKGQPxT7NOU27gXBt3q6lyZEWlIUhWm7rjN993UAmpX1obKve4rtSvi4UCaPWzpX924MblzVajW2trYA7Nq1iyZNmgBQrFgx7t+/n7bVCSGEeGdRiVH039OfEw9OYG1hzc/Vfs7awQKKAgd+hb3jdMvl2sOn08FSbvcQ5idJo+WHPy+y+ngwAP1qF2Jg3SKZ/p0Wg9//KFGiBHPnzuXgwYPs3LmTBg10PwRDQ0Nxd0/ZxQshhEh/4bHhdNrWiRMPTuBo7cgc/zlZu2nVauB/g/5rWj8erBu7Kk2rMEPxag09V55i9fFgVCr4sVlJBtXL2GNXU8vg79gJEybQvHlzfv31Vzp27EiZMmUA2Lx5s34IgRBCCNO5FXmLHjt7cD/mPjnsczDHfw7FshczdVmmo46DjV3hyt+AChr9CpW7mboqId6LoiisP3mXy/ejUjx26k4EZ+9GYmNlwYzPy9KgpLcJKjQOgxvXmjVr8ujRI6KiosiWLZt+fffu3XFwkClVhBDClM4+PEvv3b2JTIiUYAGAuAhY/QUEHwVLG2i5EIo3NXVVQrwXtUbL8I3n2Xjq7mu3cbGzYmHHSlT2Na85mt/pPRJFUTh58iQ3b96kbdu2ODs7Y2NjI42rEEKYkAQLvCTyHqxoCQ8vg60rfLEK8lczdVVCvJfYxCR6rTzFvqsPsbRQ0f6DvDjZJW/nrC0taFo2F745HE1UpfEY3LjeuXOHBg0aEBwcTEJCAnXr1sXZ2ZkJEyaQkJDA3LlzjVGnEEKIN3gxWKBarmpMrjE5awcLhF+BFS0g6h44e0P7jZCzhKmrEuK9PIlJpPOSQM6GPMXO2oJZbctTxy+nqctKVwbfnNW/f38qVqxIREQE9vb2+vXNmzdn9+7daVqcEEKIN1MUhfnn5jPyyEg0ioYmBZswo/aMrN20Bv8Di+vrmtYcRaDLDmlaRaYX8iSWVnOOcDbkKW4O1qzq9mGWa1rhHa64Hjx4kCNHjmBjkzxFJH/+/Ny7dy/NChNCCPFmLwcLdC3VlX7l+pnFncNv9SwMLv8FmsTk6xOi4dAUXbBA7krQdh04ZOHhEiJTiUlI4n/n7hMVr062XqsoLDgYxMNnCeRys2fpV5Up5Jk1I4gNbly1Wi0ajSbF+rt37+Ls7JwmRQkhhHizLB0sEHZBN3Y1Ouz12xRpAK0CwCYLX3kWmcrDZwl0XnKcC/dSzhLwXDEvZ5Z+VZmcLnbpWFnGYnDjWq9ePaZNm8b8+fMBUKlUREdHM2rUKBo1apTmBQohhEguRbDAxz/TIH8WmaM16CCsaQsJUeBeCHzKp9wmZwmo0kfmaBWZxu1HMXy5+DjBT2Jxd7Th48I5UmyT09WOXjUL4WpvbYIKMw6Dv6snT55M/fr1KV68OPHx8bRt25br16+TI0cOVq9ebYwahRBC/OtBzAN67u7J9YjrOFk7Mb3WdCp7Z405tFWXN8OfPXTDA/JVhc9Xgb2bqcsS4r2cu/uUzgGBPI5JJG92B5Z9VZn8ZjgbQFoxuHHNnTs3Z8+eZc2aNZw7d47o6Gi6dOlCu3btkt2sJYQQIm29HCww138uRbMXNXVZ6cL34S4sTy8HFPBrDC0WgnXWfbtUmIcD1x7SY8VJYhM1lPBxIaBzJTyd5ev6Td7pfRQrKyvat2+f1rUIIYR4jTPhZ+izpw+RCZHkd8nP3LpzyeWUy9RlGZ+iYLHvZ0rfXaZbrthFl3xlYWnauoRIpcfRCQTejgCUZOtDnsQxYdsVkrQK1QrlYG6HCjjZyvCWt0nVZ2jz5s00bNgQa2trNm/e/MZtmzRpkiaFCSGE0HkxWKBUjlLMqjOLbHbZ3v7EzE6TBH/1x/LMCt1ijRFY1hwGWWHWBGEWTgVH0GVJIBGx6tdu06SMD5M+K4ONlcEzlGZJqWpcmzVrRlhYGJ6enjRr1uy126lUqlfOOCCEEOLdvBgs8HGuj5lUY1LWmKM1MQbWd4br21FUFpzJ3YmS1b7BUppWkUnsufKAXitPEa/WkjubPV6vmAmgVjFPetYoiIWFfF2nVqoaV61W+8r/p6VffvmFESNG0L9/f6ZNmwZAfHw833zzDWvWrCEhIYH69esze/ZscubMehPuCiGyFkVRWHB+Ab+d/g2ApgWbMuqjUVhbZIE7imMew6rWcO8EWNmhab6Q4BtaSpq6LiFSad2JEEZsOo9Gq1CzqAez2pbHUYYBpIkMcV06MDCQefPmUbp06WTrBw4cyF9//cX69evZv38/oaGhtGjRwkRVCiFE+tBoNYw7Nk7ftHYr1Y0fq/6YNZrWiDu61Kt7J8A+G3y5GaVIFpnqS2R6iqIwa+8Nhm44h0ar0LJ8bhZ8WVGa1jSUqs/kjBkzUr3Dfv36GVRAdHQ07dq1Y8GCBfz000/69ZGRkSxatIhVq1ZRu3ZtAAICAvDz8+Off/7hww8/NOg4QgiRGbwcLDC88nDa+rU1dVnp48VgAZfc0GETeBQF9evHBwqRUWi0CmP+usiyo3cA6FmzIEPrF80aSXbpKFWN69SpU1O1M5VKZXDj2rt3bz755BP8/f2TNa4nT55ErVbj7++vX1esWDHy5s3L0aNHX9u4JiQkkJCQoF+OitIlUKjVatTp8MPv+THS41jCeOQ8mofMdh6fJT5j4IGBnAo/hbWFNeM+God/Xv9MU//7UN05hOX6DqgSnqF4+JH0+Tpw8YYXfnZnhc+DOTPn85ig1jB44wW2XXyASgXfNSxKxyr5SEpKMnVpacqY5zC1+0xV4xoUFPRexbzOmjVrOHXqFIGBgSkeCwsLw8bGBjc3t2Trc+bMSVjY62P+xo8fz5gxY1Ks37FjBw4O6XdDw86dO9PtWMJ45Dyah8xwHqO0USyNXsoD7QNssaWdfTsSLySy5cIWU5dmdD4Rxyl/Zy4qJYlHjkU55tWPpEOngdPJtssM51G8nbmdx9gkWHTVkhtRKixVCu0LafGIuMiWLRdNXZrRGOMcxsbGpmo7kw26CAkJoX///uzcuRM7u7SbbHfEiBEMGjRIvxwVFUWePHmoV68eLi4uaXac11Gr1ezcuZO6detibZ0FxqOZKTmP5iGznMdbkbfos7cPD7QPyGGfg5k1Z1IkWxFTl5UuLAIXYHF6FioUtEU/xbXZXOpZJf+dkFnOo3gzczyPYVHxdF12ihtR0TjaWjKnbVmqFHA3dVlGY8xz+Pwd8rd5p8b17t27bN68meDgYBITE5M9NmXKlFTt4+TJk4SHh1O+/H850xqNhgMHDjBz5ky2b99OYmIiT58+TXbV9cGDB3h5eb12v7a2ttja2qZYb21tna7fKOl9PGEcch7NQ0Y+j2fCz9B7d2+iEqPI75KfeXXn4ePkY+qyjE9RYPdYOPTv74yKXbBo9CsWbwgWyMjnUaSeuZzHG+HRdFwcyL2ncXg427KkcyVK+Liauqx0YYxzmNr9Gdy47t69myZNmlCgQAGuXLlCyZIluX37NoqiJGtC36ZOnTqcP38+2brOnTtTrFgxhg0bRp48ebC2tmb37t20bNkSgKtXrxIcHEyVKlUMLVsIITKcfSH7GLJ/CPGaeErnKM3MOjOzSLCAGv7qD2dW6pZrfw8fD5ZgAZFpnLwTQZelgTyNVeObw5FlX1UmT/YsML9yBmBw4zpixAgGDx7MmDFjcHZ2ZuPGjXh6etKuXTsaNEj9lCXOzs6ULJl8Vj5HR0fc3d3167t06cKgQYPInj07Li4u9O3blypVqsiMAkKITG/T9U2MOToGraLNgsECneD6DlBZQOPpUP5LU1clRKrtvvyA3qt0wQJl8rixuGNF3J1SvtMrjMPgxvXy5cusXr1a92QrK+Li4nBycmLs2LE0bdqUnj17pllxU6dOxcLCgpYtWyYLIBBCiMxKURTmn5vPzDMzgawYLPAZ3DsJVvbwWQAUbWjqqoRItXWBIYz4/b9ggdntyuNgI3O0pieDP9uOjo76ca3e3t7cvHmTEiVKAPDo0aP3Kmbfvn3Jlu3s7Jg1axazZs16r/0KIURGoNFqGH98PGuvrgV0wQJ9y/XNGvM8RtzWzdH6+IYuWKDtOshT2dRVCZEqz4MFJu24BkDL8rn5pWUprC0zRI5TlmJw4/rhhx9y6NAh/Pz8aNSoEd988w3nz59n06ZN8ha+EEK8RoImgeEHhrMreFcWDBY4/2+wwANwzQPtN+qCBYTIBDRahdGbL7L8H12wQK+aBRkiwQImY3DjOmXKFKKjowEYM2YM0dHRrF27lsKFC6d6RgEhhMhKohKj6Lu7rz5Y4JePf6Fe/nqmLit9BB2ANe0gIQo8i+uaVpcsMGuCMAvxag0D155h64UwVCoY9WlxOlX1NXVZWZrBjWuBAgX0/3d0dGTu3LlpWpAQQmRGSdokfg38lc03N6NRNCkeU2vVOFk7MaP2DCp5VTJRlens4u+wqTtoEiFfVfh8Fdi7mboqYUY0WoUJ266w+ngwGq2S5vtP0iokJmmxsbRgSpsyfFpa/ugytfcaURwdHY1Wq022Lj0m+RdCiIwkLimOIfuHsP/u/tdu4+XoxczaMymaPYu8RX5sHmwdBijg1xhaLATrtAubESJerWHQujNsOf/6NM204GpvzZz25fmoYA6jHkekjsGNa1BQEH369GHfvn3Ex8fr1yuKgkqlQqPRvOHZQghhXp7GP6X3nt6ce3gOW0tbfqr6EyVzlEyxXU7HnFlj5gBFgd1j4NBU3XKlrtBwIrwhWEAIQ0XGqem+7ATHgp5gY2nBLy1LUSl/dqMcy8PZFjtr+frNKAxuXNu3b4+iKCxevJicOXPK4GQhRJYVGh3K1zu/5nbUbVxsXJhZZyblPMuZuizT0ahhcz84u0q3LMECwggeRMXTcfFxroQ9w8nWivkdKvBRIbkamlUY3LiePXuWkydPUrRoFnm7SwghXuFaxDV67uxJeFw4Xo5ezPWfS0G3gqYuy3QSY2BdR7ixE1SW/wYLdDB1VcLM6GJWj2fJmFWhY3DjWqlSJUJCQqRxFUKYvcdxjxl3bBy3nt5K8VhoTChxSXEUcivEHP85eDl6maDCDCLmEaxq/UKwwBIomvokRSFe9DQ2kR/+vMiV+1EpHgt9GkdMooYCORxZKjGrWZLBjevChQvp0aMH9+7do2TJklhbJx+zVbp06TQrTgghTCXkWQg9dvYg+Fnwa7cp71meGbVn4Gqbha/4RNyG5S3gyU0JFhDvLfRpHF8uPs6N8OjXbiMxq1mbwY3rw4cPuXnzJp07d9avU6lUcnOWEMJsXHp8iV67evE4/jG5nHLx7QffYm9ln2wbG0sbSrqXxDIr33R0/xysbPVCsMAm8Chi6qpEJnU17BkdFx8nLCoeLxc7fmpWEkfb5G2KjZWKMrndsJLEqizL4Mb1q6++oly5cqxevVpuzhJCmJ2joUcZsHcAsUmxFMtejNl1ZuPh4GHqsjKeW/t1wQKJz8CzxL/BAt6mrkpkUsduPabbshNExSdR2NOJpV9VxsfN/u1PFFmOwY3rnTt32Lx5M4UKFTJGPUIIYTJbbm3hu8PfkaRN4gOvD5hWaxpONk6mLivjubAJfv/632CBavD5SgkWEO9s24Uw+q05TWKSlor5srGwY0XcHGxMXZbIoAxuXGvXrs3Zs2elcRVCmJXll5YzMXAiAA3yN2BctXHYWMovzxT+mQvbhgMKFG8KzedLsIDQi05IYtL2q9x+HJPiMUWrEP7Qgk2PTqGy0L1bq9EqHL7xCK0CdYvn5LcvysmcqeKNDG5cGzduzMCBAzl//jylSpVKcXNWkyZN0qw4IYQwNq2iZdrJaQRcDACgvV97hlQagoVKxtAlkyJYoBs0nCDBAkLv4bMEOi85zoV7KWcD+I8Fl58+SrH2i8p5+bFpCRm7Kt7K4Ma1R48eAIwdOzbFY3JzlhAiM1Fr1Iw8MpK/b/0NwIDyA/iq5Fcydv9lGjVs7gtnV+uWJVhAvOT2oxi+XHyc4CexuDvaMLBuEWytkjehGo2Gc+fOUbp0aSwt//uDx9vVnqqF3OX7TqSKwY2rVqs1Rh1CCJGuYtWxDNw3kCOhR7BSWTGm6hiaFJR3jFJIiIb1HeHGLl2wQJMZUK69qasSGci5u0/pHBDI45hE8mZ3YNlXlcmfwzHFdmq1GvuwszQqnyvFu7VCpJbBjasQQmR2j+Me03t3by4+voi9lT1Tak6hWq5qpi4r44l5BCs/g9BTumCB1kuhSH1TVyUykP3XHtJzxUliEzWUzOVCQKfKeDjL/KrCeN6pcd2/fz+TJk3i8uXLABQvXpwhQ4bw8ccfp2lxQgiR1kKiQvh619eEPAshm202ZtWZRSmPUqYuK+NJFiyQHdqth9wVTV2VSCNbzt/nyM2UY00NkZikZdOpeyRpFaoVysHcDhVwspXrYcK4DP4KW7FiBZ07d6ZFixb069cPgMOHD1OnTh2WLFlC27Zt07xIIYRIC5ceX6Lnrp48iX9CLqdczKs7j3wu+UxdVsZz/yysaAUx4eCaFzpsghyFTV2VSAOKojBx+1Xm7LuZZvtsWtaHX1uVwcZKbqwSxmdw4zpu3DgmTpzIwIED9ev69evHlClT+PHHH6VxFUJkSBIskEq39sGa9rpggZwlod0GCRYwE2qNluEbz7Px1F0Avqich5wu7zeVWX53R5qU8cHCQm6sEunD4Mb11q1bNG7cOMX6Jk2a8O2336ZJUUIIkZa23t7KqH9GSbDA25zfAL/3AK0a8n+sCxawczV1VSINxCYm0WvlKfZdfYilhYrxLUrRumIeU5clhMEMblzz5MnD7t27UwQQ7Nq1izx55JtACJGxHI4/zNYjWwFomL8hP1X7SYIFXuWfOf8GCwDFm0GL+WAlN9mYg8fRCXy1JJCzdyOxs7Zgdrvy1C6W09RlCfFODG5cv/nmG/r168eZM2f46KOPAN0Y1yVLljB9+vQ0L1AIId7m4qOL/HP/nxTrbz69ydZ4XdMqwQKvoSiwaxQc/vfnd+Xu0OAXCRbIAEKexLLtQhhJWuWd96GgsP7EXYIexeDmYM3iTpUonzdbGlYpRPoyuHHt2bMnXl5eTJ48mXXr1gHg5+fH2rVradq0aZoXKIQQb/LHjT8YfWQ0GuX14Sf9yvaja+muMsH5y14OFqgzEqoNkmCBDOB40BO6Lg0kKj4pTfaXy82epV9VppCnDJERmds7zVvRvHlzmjdvnta1CCFEqimKwqILi5h+Snel8AOvD/B2Sn4TkaJVcH7gTKfinaRpfVlCNKz7Em7ulmCBDGb7xTD6rj5NYpKW4t4ulPBxea/9uTlY0/XjAu99I5YQGcE7T7iWmJhIeHh4iiStvHnzvndRQgjxJhqthomBE1l1ZRUAnUt2ZkD5ASmGAajVarZs2WKKEjO26Iew6jMIPS3BAhnMin/uMPLPC2gV8PfLycy25bCzlmEbQjxncON6/fp1vvrqK44cOZJsvaIoqFQqNJrXv10nhBDvK1GTyIiDI9hxZwcAQysNpUPxDiauKhN5EgQrWsCTWxIskIEoisLUXdeZsfs6oJuq6semJbGylDHZQrzI4Ma1U6dOWFlZ8ffff+Pt7S1vvwkh0s2zxGf039ufwLBArCys+LnazzT0bWjqsjIPCRYwucv3o7geHp1i/YFrD9lwUje/av86hRngX1h+vwrxCgY3rmfOnOHkyZMUK1bMGPUIIcQrhceG03NXT65FXMPR2pFptabxofeHpi4r87i5F9a2h8RoyFlKd6VVggXSVcDhIMb+fQnlNZMEWKhgbNOStP9Q0tyEeB2DG9fixYvz6NH75RsLIYQhgiKD6LGzB6ExobjbuTPHfw5+7n6mLivzkGABk3o5ZrVMHjccbZKPW7WxsuDLKvlkflUh3sLgxnXChAkMHTqUn3/+mVKlSmFtbZ3scReX97v7UQghXnTu4Tl67+7N04Sn5HPJx1z/ueR2zm3qsjKPo7Nh+wjd/0s0h+bzJFggHak1WoZtPMemU/cAGFK/KL1qFpRhAEK8I4MbV39/fwDq1KmTbL3cnCWESGsH7h5g8P7BxCXFUdK9JLP8Z5HdLrupy8octFpdsMCRGbrlyl//GywgN/ukl5gEXczq/msSsypEWjG4cd27d68x6hBCiGR+v/47Y46OQaNoqJqrKlNqTMHB2sHUZWUOGjX82QfOrdEt1xkF1QZm6WCBuxGxPIhKSLfjabQK4/53SWJWhUhjBjeuNWrUMEYdQggBpAwWaFKwCaM/Go21hfVbnikAXbDA+o5wY9e/wQK/Qbl2pq7KpBYcuMW4LZdNcuxsDtYskphVIdLMOwcQxMbGEhwcTGJiYrL1pUuXfu+ihBBZk0arYULgBFZf0UWQdi7ZmYHlB8p4wNR6MVjA2gE+WwpF6pm6KpPRahV+3nKZhYeCAF3sqaVF+n0tebva8XOLUhT0kJhVIdKKwY3rw4cP6dy5M1u3bn3l4zLGVQjxLhI0CXx78Ft23NmBChVDKw2lfXGJIE01CRZIJjFJy+D1Z9l8NhSAbxsVo9vHBeSPICEyOYNH6Q8YMICnT59y7Ngx7O3t2bZtG0uXLqVw4cJs3rzZGDUKIczcs8Rn9NzVkx13dmBtYc3E6hOlaTXE/bOwqJ6uaXXLC112Zumm9Vm8mq+WBLL5bChWFiqmtilD9+pyJ78Q5sDgK6579uzhzz//pGLFilhYWJAvXz7q1q2Li4sL48eP55NPPjFGnUIIM/VysMD0WtP5wPsDU5eVedzcC2s7QOIzXbBA+w3g7GXqqtLFk5hE1BptsnXRCUn0W32ai6FRONhYMqd9BWoU8TBRhUKItGZw4xoTE4OnpycA2bJl4+HDhxQpUoRSpUpx6tSpNC9QCGG+XgwWyGGfgzn+cyiWXVL5Uu3FYAHf6tBmRZYIFohXaxi64Zx+GMCruDvaENC5EqVzu6VfYUIIozO4cS1atChXr14lf/78lClThnnz5pE/f37mzp2Lt7fEBwohUkeCBd5TFg0WiIpX033ZCf659QTglTdbFfNyZlbb8uTP4Zje5QkhjMzgxrV///7cv38fgFGjRtGgQQNWrlyJjY0NS5YsSev6hBBm6MDdA3yz7xviNfGUylGKmXVmSrBAar0cLPBBT6j/c5YIFngQFU/X5ae5EvYMJ1sr5neowEeFcpi6LCFEOjK4cW3f/r8bJipUqMCdO3e4cuUKefPmJUcO+QEihHizF4MFquWqxuQakyVYILU0avizN5xbq1v2Hw1VB2SJYIEHcdB6/nFCI+PxcLZlSedKlPAx/2ERQojk3nke1+ccHBwoX758WtQihDBjiqKw8PxCZpzWXSmUYAEDJUTDui/h5m5dsEDTmVC2ramrShenQ54y/YIlMUnx+OZwZNlXlcmTXf7YESIreu/GVQgh3kaj1fDL8V9Yc1UXQdqlZBf6l+8v0xOl1svBAq2XQeG6pq4qXey+/IDeq04Rn6SidC4XAjpXxt3J/MfyCiFeTRpXIYRRJWgSGHFwBDvv7JRggXfx5BYsbwERQVkuWGBdYAgjfj+PRqvg56Zl+VcVcXWUplWIrEwaVyGE0UQlRtF/T39OPDiBtYU1P1f7mQa+DUxdVuYRegZWtoKYh7pggfa/Q45Cpq7K6BRFYdbeG0zacQ2A5uV8+NgmGAcb+ZUlRFZn/rehCiFMIjw2nE7bOnHiwQkcrR2Z4z9HmlZD3NwLSz7RNa1epXRpWFmgadVoFUZtvqhvWnvWLMiE5iWwlN9WQgjesXE9ePAg7du3p0qVKty7dw+A5cuXc+jQoTQtTgiROd2KvEX7Le25HnGdHPY5WNJgiaRhGeLcelj5GSRG64IFOm3JEmlY8WoNfVadYtnRO6hUMKpxcYY1KCZjoYUQegY3rhs3bqR+/frY29tz+vRpEhISAIiMjOTnn39O8wKFEJnL2Ydn+XLrl9yPuU8+l3wsb7hc0rAMcXQWbOqqS8Mq0QLabQA7F1NXZXSRcWo6Lj7O1gth2Fha8NsX5ehc1dfUZQkhMhiDG9effvqJuXPnsmDBAqyt/5vGpmrVqhL5KkQWtz9kP123dyUyIZJSOUqxrOEyScNKLa0WdnwP27/VLX/QE1ouyhJpWGGR8bSZd5RjQU9wsrViSedKfFrax9RlCSEyIINHul+9epXq1aunWO/q6srTp0/ToiYhRCYkwQLvISlRFyxwfp1u2X8MVO2fJYIFboQ/o+PiQO49jZNgASHEWxncuHp5eXHjxg3y58+fbP2hQ4coUKBAWtUlhMgkFEVh/rn5zDwzE4CmBZsy6qNREiyQWgnP/g0W2AMWVtBkJpT9wtRVpYuTdyLosjSQp7FqCRYQQqSKwY1rt27d6N+/P4sXL0alUhEaGsrRo0cZPHgwP/zwgzFqFEJkUBqthvHHx7P2qi6CtGuprvQr109upkmt6HDdTVj3z/wbLLAcCvubuqp0oQ8WUGspk8eNxR0rSrCAEOKtDG5chw8fjlarpU6dOsTGxlK9enVsbW0ZPHgwffv2NUaNQogM6OVggWGVh9HOr52py8p4khJg7zjdnKwve3QdnoWCgzu0XQ+5K6R7eYZQFIU1gSFsOX8fraK88360Wjh++wkarULNoh7Mblde5mgVQqSKwT8pVCoV3333HUOGDOHGjRtER0dTvHhxnJycjFGfECIDikqMot+efpx8cFIXLPDxzzTIL3O0phAfCWvawe2Dr9/GLR90+B3cC6ZfXe9Aq1UYt+Uyiw4Fpdk+W1XIzfgWpbCWSVqFEKn0zn/i2tjYULx48bSsRQiRCTyIeUDP3T25HnEdJ2snpteaTmXvyqYuK+N5FgYrWsGD82DjDP6jwD5b8m0srKBATbB3M0WFqZaQpGHw+nP8dTYUgN61ClIkp/N77dPD2ZYqBdxlWIkQwiAGN661atV64w+aPXv2vFdBQoiM69bTW/TY1YP7MffxsPdgjv8cimYvauqyMp5H12F5C4gMBkdPaL8BvMuYuqp38ixeTY8VJzl84zFWFiomfVaGZuVymbosIUQWZXDjWrZs2WTLarWaM2fOcOHCBTp27JhWdQkhMpgz4Wfos6cPkQmR5HfJz9y6c8nlJA1MCndP6G64insC2QtC+42QPXNOpB/+LJ7OAYFcDI3CwcaSue0rUL2Ih6nLEkJkYQY3rlOnTn3l+tGjRxMdHf3eBQkhMp79IfsZvH8w8Zp4Sucozcw6M8lml+3tT8xqru2A9R1BHQs+5aHdenDMYbTDRSckMf/ALcIi44yy/yM3H3M3Ig53RxsCOleidG43oxxHCCFSK81u42zfvj2VK1dm0qRJabVLIUQG8GKwwMe5PmZSjUkSLPAqp1fC5r6gaKCQP3y2FGyNd9Pqw2cJdF5ynAv3oox2DIB87g4s7VyZ/DkcjXocIYRIjTRrXI8ePYqdnV1a7U4IYWISLJBKigKHpsDusbrl0p9D05lgabzP0+1HMXy5+DjBT2Jxd7Sh00f5sbBI+5ucHGwsaVo2F9kdbdJ830II8S4MblxbtGiRbFlRFO7fv8+JEyckgEAIM/FysEC3Ut3oW66v3AH+Mq0Gto2A4/N0y1UHgP9oo0a1nrv7lM4BgTyOSSRvdgeWfSVXQ4UQWYfBjaura/IMaQsLC4oWLcrYsWOpV69emhUmhDCNBE0Cww8MZ1fwLlSoGF55OG392pq6rIwnKQE2dYdLfwAqaDAePuxp1EMeuPaQHitOEpuooYSPCwGdK+HpLO90CSGyDoMaV41GQ+fOnSlVqhTZssmNGUJkBpcfX+bA3QMopC7p6EjoEU6Hn8bawppfPv6FevnlD9IUXgwWsLCGFvOgZMs02XXIk1j+PncftUabbH10QhKLDwWRpFWoVigHcztUwMlW0qaEEFmLQT/1LC0tqVevHpcvX5bGVYhMYGvQVr499C1J2iSDnudk7cSM2jOo5FXJSJVlYlH3YWUreHBBFyzw+UooUCNNdn0qOIIuSwKJiFW/dpsmZXyY9FkZbKwkbUoIkfUY/Od6yZIluXXrFr6+mXNeQiGyihWXVjAhcAIAlb0qk88lX6qeZ2tpS6sirSjolrEjSE3ixWABp5zQbgN4l06TXe+58oBeK08Rr9bi5+1CubxuKbYp7u1C28p5jXIjlhBCZAYGN64//fQTgwcP5scff6RChQo4Oia/KcDFxSXNihNCGE5RFKaemkrAhQAAvij2BcMqDcPSwtLElWVyLwcLdNgE2fKnya7XBYYw4vfzaLQKNYt6MLtdeRxsZBiAEEK8zOCfjI0aNQKgSZMmye4wVhQFlUqFRqNJu+qEEAZRa9WMPjKazTc3A9C/fH+6lOwiswG8r2vbYV1HSIpL02ABRVGYtfcGk3ZcA6Bl+dz80rIU1pYyDEAIIV7F4MZ17969xqhDCPGeYtWxDNo3iMOhh7FUWTL6o9E0K9TM1GVlfqdXwOZ+7xUscPl+FJdCUwYFHA96wtoTIQD0rFmQofWLyh8ZQgjxBgY3rr6+vuTJkyfFD1dFUQgJCUmzwoQQqfck/gm9d/XmwuML2FvZM6nGJKrnrm7qsjI3RYGDk2HPj7rlMl9Ak98MDhZY8c8dRv55Ae1rJnVQqWDkp8XpXFXuGxBCiLd5p8b1/v37eHp6Jlv/5MkTfH19ZaiAEOks5FkIPXb2IPhZMG62bsyqM4vSHmlzw1CWpdXA1mEQuEC3XLU/+I8xKFhAURSm7rrOjN3XASiX1w0Xu+RNr7WlijaV8lK3eM40K10IIcyZwY3r87GsL4uOjpbIVyHS2eXHl+m5qyeP4x+TyykXc/zn4OsqV+7eizoefu8Ol/7kXYMFkjRavv/jAmsCde9C9a9TmAH+hWUYgBBCvKdUN66DBg0CQKVS8cMPP+Dg4KB/TKPRcOzYMcqWLZvmBQohXu1o6FEG7htIjDqGotmKMsd/Dh4OHqYuK3OLj4TVbeHOIbC0geZzDQ4WiEvU0Hf1aXZdfoCFCsY2LUn7D1M3FZkQQog3S3Xjevr0aUB3xfX8+fPY2NjoH7OxsaFMmTIMHjw47SsUIgt7lviMK0+upFh/6+ktfgn8hSRtEpW9KjOt1jScbZxNUKEZSYNggaexiXRZeoKTdyKwsbJgxuflaFDSy0gFCyFE1pPqxvX5bAKdO3dm+vTpMl+rEEZ24dEFeu3qRURCxGu3qZ+/Pj9X+xkbS5vXbiNS4eE1WNHyvYIF7j2No+Pi49wIj8bFzopFnSpRKX92IxUshBBZk8FjXAMCAoxRhxDiBYfuHWLQvkHEJcXhbueOq61rssdVqKibvy49y/TEQiVzfr6XkEBY1fq9ggWuhj2j4+LjhEXF4+Vix9KvKlPUS66ACyFEWpNoFiEymM03NzPq8CiSlCQ+8vmIKTWn4Gjt+PYnCsO9GCyQqwK0XWdwsMCxW4/puuwEz+KTKOzpxNKvKuPjZm+kgoUQImuTxlWIDEJRFAIuBjD15FQAPinwCT9+9CPWBs4bKlLp1HL4q/+/wQJ1ofVSsDHsD4RtF8Lot+Y0iUlaKubLxsKOFXFzkGEbQghhLNK4CpEBaBUtvwb+yorLKwDoVKITAysMlGEAxqAocHAS7PlJt1ymLTSZ8V7BAnWL5+S3L8phZ21phIKFEEI8J42rECaWqEnku0Pfse32NgAGVxxMxxIdTVyVmXo5WKDaIKgz8r2CBb6onJcfm5bAylL+yBBCCGOTxlUIE4pOjGbA3gEcCzuGlYUV46qOo1GBRqYuyzylCBb4BT7sYdAuXg4WGOBfmP51JFhACCHSizSuQpjIo7hH9NzVkytPruBg5cC0WtOo4lPF1GWZp/hI2NDxhWCBeVCyhUG7eDlY4KdmpWj7QV4jFSyEEOJVpHEVwgRuR96mx64e3Iu+R3a77Mzxn0Nx9+KmLsss2akjsFreGMIvga2LLljAt7pB+3gxWMDWyoIZX5SjfgkJFhBCiPQmjasQ6ez8w/P03t2biIQI8jrnZa7/XPK45DF1Webp0XU+vjoWlfqxLlig/UbwKmXQLiRYQAghMg5pXIVIRwfvHuSb/d8QlxRHcffizK4zG3d7d1OXZZ5CjmO1qjXW6giU7AVRdfgdsuUzaBdXwqLouPg4D6IS8HbVBQsUySnBAkIIYSrSuAqRTv688SejjoxCo2j4yOcjptacioO1g6nLMk9Xt8H6TqiS4njiUBDnjluwdjXsrX0JFhBCiIxHGlchjExRFBZdWMT0U9MB+LTAp4z9aKwECxjLC8EC2oL+HHFsQ30Hw65qb7twn35rzpCYpKVS/mws/LISrg5yvoQQwtRk4kEhjEiraJkQOEHftHYu0Zlx1cZJ02oMigL7f4XNfXRpWGXboflsORpLW4N2s/yfO/RceYrEJC31iudkeZcPpGkVQogMQq64CmEkiZpEvj30LdtvbwdgSMUhfFniSxNXZaa0Gtg6FAIX6pY//gZq/wBJSanehaIoTNl5jd/23ACg7Qd5+bFpSSwtZI5WIYTIKKRxFcIIniU+Y8DeARwPOy7BAsamjodN3eDyZkAFDSfAB18btIskjZbvfr/A2hO6YIGB/kXoV6eQBAsIIUQGI42rEGnsYexDeu7qydWIqxIsYGxxT2FNW7hz+L2CBfqsOsXuK+ESLCCEEBmcSce4jh8/nkqVKuHs7IynpyfNmjXj6tWrybaJj4+nd+/euLu74+TkRMuWLXnw4IGJKhbizW5H3qbD1g5cjbiKu507SxoskabVWKJCIaChrmm1ddHN0Wpg0xoRk0i7hf+w+0o4tlYWzG1fQZpWIYTIwEzauO7fv5/evXvzzz//sHPnTtRqNfXq1SMmJka/zcCBA/nrr79Yv349+/fvJzQ0lBYtDPvlJER6OPfwHF9u/ZJ70ffI65yX5Y2W4+fuZ+qyzNPDq7Cwri4Ny8kLOm8xOA3rbkQsreYe4VTwU1ztrVnZ9QPqSRqWEEJkaCYdKrBt27Zky0uWLMHT05OTJ09SvXp1IiMjWbRoEatWraJ27doABAQE4Ofnxz///MOHH35oirKFSOHFYIES7iWYVWeWBAsYS/AxWNUa4p+CeyFov0mCBYQQIovIUGNcIyMjAcieXRenePLkSdRqNf7+/vptihUrRt68eTl69OgrG9eEhAQSEhL0y1FRUQCo1WrUarUxy9cf58V/ReZkyHn869ZfjD02Fo2ioYp3FX6t9isOVg7yNWAEqmvbsPy9G6qkOLQ+FdC0WQUO7vCKz/Xj6ATGb73KhSBLlt07luxGqyth0UQnJFHY05FFX1bA29VOzlcGJj9XzYOcx8zPmOcwtftUKYqipPnR34FWq6VJkyY8ffqUQ4cOAbBq1So6d+6crBEFqFy5MrVq1WLChAkp9jN69GjGjBmTYv2qVatwcJCUIpF2FEXhQMIBdsbvBKCsdVmaOzTHUmVp4srMU95H+ygbEoAKhTCXMpzI3+e1c7Q+ioc5ly15FP/6WQEKOCt0K6bBIUP9+S6EEFlTbGwsbdu2JTIyEhcXl9dul2F+ZPfu3ZsLFy7om9Z3NWLECAYNGqRfjoqKIk+ePNSrV++Nn4i0olar2blzJ3Xr1sXaWiYtz6zedh61ipZJJyex85quae3o15F+ZfvJ9EnGoChYHJ6C5enFAGhLt8W90WTqvybE4WJoFD8uP8Wj+ERyudlRxyOG8mVLY2n53487BxtLPvTNjo2VZLBkBvJz1TzIecz8jHkOn79D/jYZonHt06cPf//9NwcOHCB37tz69V5eXiQmJvL06VPc3Nz06x88eICX16tvorC1tcXWNuVVGGtr63T9Rknv4wnjeNV5TNQk8t2h7/TBAkMrDaVD8Q6mKM/8aTWwZTCc0DWtfDwYi9rfY/GaPxAOXX/E18tPEJOowc/bhYUdynHi4G4alc4l349mQH6umgc5j5mfMc5havdn0ssNiqLQp08ffv/9d/bs2YOvr2+yxytUqIC1tTW7d+/Wr7t69SrBwcFUqSJTDIn09yzxGT129WD77e1YWVgxsfpEaVqNRR0P6778t2lVQcNfoc4P8Jqm9c8z9+i85DgxiRo+KujO2q8/xNPZsLhXIYQQGZtJr7j27t2bVatW8eeff+Ls7ExYWBgArq6u2Nvb4+rqSpcuXRg0aBDZs2fHxcWFvn37UqVKFZlRQKS78Nhweu7qybWIazhaOzKt1jQ+9JavQ6OIi4DVbSH4iC5YoMUCKNHstZsvPHiLn/53GYBPSnszpXUZbK0s5SYQIYQwMyZtXOfMmQNAzZo1k60PCAigU6dOAEydOhULCwtatmxJQkIC9evXZ/bs2elcqcjqgiKD6LGzB6ExobjbuTPHf47M0WoskfdgZSvdHK22Lvy/vfuOjqJu2zj+3WwqgSR0CKEElC5Il6KgoGBHUBFBqgWBRyAoRcX6KqCiWBAUhKgIPCooSlOkCQhJCEVKKNJCSQg9Cem7v/ePlTxEggZJstnN9TmHc5zZ2Zl7vYHczM7MxSNzIPTmXDe12w0Tlu3m018PANCvTQ1euqc+Hh661lhExB05dXDNywMNfH19mTJlClOmTCmEikQu9/vJ3xmyYgjn0s9RrVQ1pt0+jaqlqjq7LPeUsBtmd4fEo45ggd7zoVLDXDfNyLIzev7vfLflGACjutTh6fa1dIOciIgbKxI3Z4kUVWuPrWXM+jGkZqXSsGxDpnSaQhnfMs4uyz3lCBa4Hh5bAEG5x69eSM9i0Oxo1u47hdXDwsTujXiwWUiu24qIiPvQ4CpyBdHp0fzw6w/YjI22Vdrybvt3KeGlZwEXiN1L4Nv+kJUGIS3g0a+hRO7/QDiVnM6A8Ch+P3oePy8rH/duyq11KhRywSIi4gwaXEX+whjDzJ0z+S71OwDuq3Ufr7R5BS8PPb6lQER/DouGg7HD9Z3hoVmke/jy9YZDnEhMv2zzRb8f59DpFMr4ezOzXwturBpU6CWLiIhzaHAVuYTNbmNi1ETm7p4LQL/6/QhrHqbrJguCMfDr27DqDcdyk95wz/skZhqeCo9iw4HTV3xrSGk/vhjQkprlSxZSsSIiUhRocBX5U7otnefXPs/Ph38G4C7fu5SGVVByCRbgthdJSEqn76woYuIS8fe20q1pCNa/PCEgwNeT3q2rU6GUrxMKFxERZ9LgKoIjWGDYqmFExUfh6eHJ6ze9jm2XzdlluafMVJj/OOxeBFjgrreh5RMcOJlMn5mRHD2bSrmSPoT3b0HDKoHOrlZERIoQBXVLsZeQkkC/Zf2Iio/C38ufaZ2m0blGZ2eX5Z5Sz8KX3RxDq9UbHv4cWj7BltizPDhtA0fPplKjbAkWPN1GQ6uIiFxGZ1ylWLtSsIASlwrA+WOOZ7SejAGfQOg5B2q0Y9WeBAbP3kxqpo1GIYHM7NeCciUV1SoiIpfT4CrF1raT2xi6Yijn0s9RPaA60zpNI6SUngVaIBJ2w+xukHgMSlV2BAtUbMC30UcZPf93bHbDLbXLM7VXU/x99NeSiIjkTj8hpFj69eivjFw9kjRbmoIFClrsRpjTwxEsUK429J6PCazK1NV/8NayPQB0a1KFiQ82wsuqq5dEROTKNLhKsfPdvu94dcOrChYoDLsXw7cDcgQL2H1L89qPuwj/7RAAT7WvyZgudfX0BhER+UcaXKXYMMYwfft0PtzyIaBggQIXHQ6LRjiCBWp3gQdnke7hQ9i8LSz+PQ6AcffUZ2C7UOfWKSIiLkODqxQLNruNCZETmLdnHgADGg5geNPhOstXEIyBNW/B6jcdy5cECzwZHsnGA2fwslqY9PCN3Nc42Lm1ioiIS9HgKm4v3ZbO2LVjWX54ORYsjGoxit71ezu7LPdkt8HikRA9y7F8y3Nw6wucSEqn78xIdscnUdLHk08ea0bb68o5t1YREXE5GlzFrSVmJDJs5TA2ndiEl4cXb7Z7ky6hXZxdlnu6QrDA/pPJ9PkskmPnFCwgIiLXRoOruK2ElAQG/TKIfWf34e/lz/u3vk+ryq2cXZZ7SjkDc3vCkY1g9YHu06H+/WyJPcuA8CjOpmRSo2wJvhjQimpldSOciIj8OxpcxS0dOH+AQcsHEXchjnJ+5ZjaaSp1y9R1dlnu6fzRP4MFducMFtidwOCvHMECjUMC+UzBAiIico00uIrb2XZyG0NWDOF8+nkFCxSw9OM7sM55EM/kOLL8KxF372wy/euxMSKWcQt3YLMb2tcuz8cKFhARkXygnyTiVtYcWcOza54lzZbGDeVu4KOOHylYoIDERPxM8NJ+BHKBP+zB9D09mmPhCUBC9jbdmlZhYncFC4iISP7Q4Cpu49JggXZV2jGp/SQFCxSQLT/Ppt764fhaMtliavOMx2iSfEsR8OfrXlYPet1UnRGdrtcjx0REJN9ocBWXp2CBwhXxzSSa73gdq8WwtURr6g75hrX+pZxdloiIFAMaXMWl/TVY4PEbHueZJs/oLF8BMHY7G8NH0zr2U7BAVOm7aTI4HE8vb2eXJiIixYQGV3FZfw0WGN1yNL3q9XJ2WS7Dbjdk2u152tbYbGz75HFan1kIwMaQgbQa8A4WD127KiIihUeDq7iky4IFbn6TLjUULJBXO46d5+mvojlyJvUft/Uhgw+8PqKzdRN2YyGqwfPc9PCoQqhSREQkJw2u4nJOXDjB0yuezg4W+ODWD2hZuaWzy3IZa/edZNCX0VzIsP3jtoEkM8P7HVp47CUdL3a1mUSrzn0LoUoREZHLaXAVl3Lg3AEG/fK/YIFpnaZRp0wdZ5flMr7fcoxnv9lGlt3Q9rqyTO7RBF+v3L/utyQew+/rh7Ge2ovxCcT6yByahLYr5IpFRET+R4OruIytCVsZunIo59PPUyOgBtNun0aVklWcXZbLmP7rAd5YEgPAfY2Deeehxnh7XuEa1YQYRxpW4jEoFYyl93w8K9YvxGpFREQup8FVXMKlwQKNyjXio44fUdq3tLPLcgl2u+HNJTHMWHcQgAFtQ3nx7np4eFzhyQuHN8DcHpB2HsrVgd7zIahqIVYsIiKSOw2uUuhSMlN4fePrbDi+Ic/vOZt+Fruxc3OVm3mn/TvFOlhg74kkRn37O0fP/vONVQA2u52zKZkAPH9XXZ64ueaVHxcWswjmD4SsNKjaCnrOgxJKHhMRkaJBg6sUqjNpZxjyyxB2nN5x1e/tel1XXmr9UrEOFth06AwDwqNITMu6qvd5WS1M7N6Ibk1D/mbnM2HxSDB2qH0nPDgTvIvvPxBERKTo0eAqheZo0lEG/TKIw4mHCfIJ4o12b1CxRMU8vdffy5+QUn8zdBUDP+2M55m5W0jPstO0WhCv3tcQT2veghYqBvhSxv8KQQHGwOoJsGaCY7lpH7j7PbDqrwcRESla9JNJCkXM6RgGrxjMqdRTBPsHM+32aYQGhjq7LJcxJyKWF7/fjt1Ax7oV+OjRpvh5W699x7YsWDISosMdy+1HQ4exoOQxEREpgjS4SoGLiItg2KphXMi8QO3StZnaaSoVSlRwdlkuwRjD+yv2MfmXfQD0aF6VNx5oiKc1HxKrMlPh24GwZzFggbsnQYuB175fERGRAqLBVQrUsoPLGLtuLFn2LFpUasH7t75PKe9Szi7LJdjshnELdzAnIhaA/9x2HWG3177yjVVXI+UMzO0JRzaC1Qe6z4D69137fkVERAqQBlcpMLN3zWZi1EQA7qh+B+NvHo+39QrXWUoOaZk2npm7hZ93ncBigdfua8BjrWvkz87PH3U8o/XkbvANdDw5oHqb/Nm3iIhIAdLgKvnOGMN7m99j1o5ZAPSs25PRLUZj9ciHazKLgXMpGTz++SY2HT6Lt6cH7/e4kTtvqJw/Oz+xyzG0Jh2HUsHQ+1uo2CB/9i0iIlLANLhKvsq0Z/LKb6/ww/4fABjWdBgDGw7Mn6+3i4Hj51LpOzOSfQnJlPL1ZEaf5rSqWTZ/dn74N5j7iIIFRETEZWlwlXyTkplC2Oow1h9fj9Vi5ZU2r9D1uq7OLstl7D2RRJ/PIolPTKNSgC/hA1pQt1JA/uw85kfHjVi2dAULiIiIy9LgKvni0mABP08/3mn/DreE3OLsspzGbjeE/3aI6NizeX7P2r0nSUzLolZ5f74Y2IoqQX75U0zUZ7DkWUewQJ27HMECXvm0bxERkUKkwVWu2ZGkIwxaPojYpFiCfIKY0nEKjco3cnZZTpORZee5b7excOvxq35v02pBfNa3BaWvFBZwNYyBVW/Cr2/9ufO+cPe7ChYQERGXpZ9gck1iTsfw9C9PczrttIIFgOT0LAZ9Gc26P07h6WFhcIdalC3pk6f3Bvh5cmfDyvh65VOwwOIRsPkLx7KCBURExA1ocJV/bcPxDYxYPYILmReoU7oOUztNpXyJ8s4uy2lOJqXTPzySHccSKeFt5eNeTelQxwlBCxkpMH8g7FkCFg9HsEDzAYVfh4iISD7T4Cr/ypIDS3hh/Qtk2bNoWaklk2+dXKyDBQ6dukCfmZHEnkmhrL83M/u1oHHVoMIvJOWM48kBRyLA0xe6fwb17in8OkRERAqABle5al/u+pK3ohzXTXau0Zk3271Z5IMFTienMzcylqS0rDxtb7PbOXDYg+0/7cXq8ffxqgZYsPkop5IzqFrGjy8GtCK0nH8+VH2Vzh1xPKP11J4/gwX+C9VbF34dIiIiBUSDq+SZ3diZHD2ZWTsdwQKP1n2U0S1H42H5+8HO2Q6fdpwNPXw65Srf6cHK44fyvHWD4ABm9W9BhVK+V3mcfHBpsEBAFcczWivUK/w6RERECpAGV8mTTHsmL61/iUUHFgGuEyyw/eh5+odHZp8NvbNh3hKobDYbBw8cJLRmKFbrP98sVcbfm16tqlHK1+taS756h9bD3J6Q/mewwGMLIDCk8OsQEREpYBpc5R+5arDA2n0nGfRlNBcybNSvHED4gLyfDc3MzGTJkv3c1aUOXl5OGEbzKkewwE3Qc66CBURExG1pcJW/dTr1NENWDGHn6Z34efoxqf0kbg652dll/aPvtxzj2W+2kWU3tL2uLNN6N3PO2dCCFDUDFj8LGAULiIhIsaDBVa6oIIMF9sQnEXHwdL7s66+OnElh+tqDANzbOJh3HmqEj2c+PBu1qPhrsECzfnDXJAULiIiI29NPOsnVpcECVUpWYVqnadQIrJEv+1641XE2NNNm8mV/VzKgbSgv3l0PD4+ifR3uVbksWGAMdBijYAERESkWNLjKZQoyWGDG2gP83+IYwBFvWimwYO7Av7VOBR5sFlLkbx67Khkp8O0A2LtUwQIiIlIsaXCVHC4NFmhVqRXv3fpevgQL2O2GCct28+mvBwDo37YG4+6u715nQwuSggVEREQ0uMr/fLHzC97e9DaQv8ECGVl2Rn27je+3HgdgzJ11eeqWmu51NrQgKVhAREQE0OAqXB4s0KteL0a1GJUvwQLJ6Vk8PTuatftO4elhYWL3RnRvpmeM5tmJnX8GC8QpWEBERIo9Da7F3F+DBUY0G0H/Bv3z5WzoyaR0BoRHsf3YeUp4W/m4V1M61KlwzfstNi4NFihf1zG0KlhARESKMQ2uxdhfgwVebfMq9193f77s+9KY1TL+3szq14LGVYPyZd/Fwq4fYP7j/wsWeHQe+JV2dlUiIiJOpcG1mCrIYIG/xqx+MaAVoeX882XfxcKlwQJ174HuMxQsICIiggbXYunSYIHSPqWZ0nEKN5S/IV/2fS0xq8WeMbDqDfjVcYMczfo7Hnnl4UbhCSIiItdAg2sxEx23nWGrh3A+4ywV/CrzYrP38LZVY0980jXve+uRs7z4/Q4ybW4cs1pQbFmwaDhs+dKx3GEstB+tYAEREZFLaHAtRj6NWsqHO8aBRzq2tMoc2DuA/psPAYfy9ThuGbNakP4aLHDPe44YVxEREclBg2sx8dqq2Xx9+B0sHjZIvQ6/0wMo4Ze/X+FbPSw81KwqYbfXVrBAXqWcgTk94GikI1jgwZlQ925nVyUiIlIkaXAtBgb/+B5rz8zEYoHyllYs7P8hpXx0s4/TnTsCs7vBqb3gGwSP/heq3eTsqkRERIosDa5uLMtm45H5L7In1fGM1ut87uSbh8bjadVX+E6XI1gg5M9ggbrOrkpERKRI0+Dqpi6kp9P162eIt/8GQOvSfZh2z0g8PK49DUuu0aF1MPfRP4MF6v0ZLFDF2VWJiIgUeRpc3dCJ5PM88M2TJHnswhgPHggZweud+jm7LAHYtRDmP+EIFqjWBnrOUbCAiIhIHmlwdTN7Th7n0R+fIMMai7F7Mbj+awxudY+zyxKAyOmw5DkULCAiIvLvaHB1Ixtj9/DU8kHYPU+BzZ9XW71L9wZtnF2WGAMr/w/WvuNYbj4A7npHwQIiIiJXSYOrm1i4K4IXNw4Hz2Q8ssrycaeptK1ez9lliS0LFg2DLbMdy7e+ALc8p2ABERGRf0GDqxv4NGopH+wYh8WajldWCHPum07d8iHOLksyUuDb/rB3mYIFRERE8oEGVxex/2QyI7/exsFTF3KsN/6bsZebh8XDRkl7Pb57+BMqldLNPk6XcgbmPAxHo/4MFpgFde9ydlUiIiIuTYOrC9gSe5YB4VGcTcnMsd6rzFp8yy/GAlTwuInvH/lAwQJFwblY+LIbnN73Z7DA11CtlbOrEhERcXkaXIu4VbsTGPzVZlIzbTQOCWR8t0Z4ecLnuz9i4cHFADxQ8xFeaTcWD4ue0ep08TscwQLJ8Y5ggccWQPk6zq5KRETELWhwLcK+2XSEMQu2Y7Mb2tcuz8e9muLtaXhx/YssObgEgJHNRtK3QV8sutnH+S4NFqhQH3p9q2ABERGRfKTBtQgyxvDx6v28/dMeALo1rcLE7o3IsKcyZMUINsRtwNPiyWttX+PeWvc6uVoBYOf3sOAJsGUoWEBERKSAaHAtYmx2w+uLdhH+2yEAnu5Qi1Gd63A67TSDfxlMzJkY/Dz9eK/De7St0ta5xYpDxKewdBQKFhARESlYGlyLkLRMGyO/3sbi7XFYLPDSPfXp3zaU2MRYnlr+FEeTj1LGtwxTOk6hYbmGzi5XjIGVr8PaSY7l5gPhrrcVLCAiIlJANLgWEYlpmTz5xSY2HjiDt9WDSQ835t7Gwew8vZPBvwzmTNoZqpSswie3f0L1gOrOLldsmfDjcNh6MVjgRbjlWQULiIiIFCANrkXAicQ0+s6MZHd8EiV9PPn0sWa0ua4cvx37jeGrh5OalUq9MvX4uNPHlPMr5+xyJeMCfNMf9v30Z7DAZGjW19lViYiIuD0Nrk62/2QyfT6L5Ni5VMqX8iG8fwsaBAey6MAixq0bR5bJ4qbKNzH51sn4e/k7u1y5cNoRLHBsE3j6wUOzoM6dzq5KRESkWNDg6kSbY88y8M9ggdBy/nwxoCVVy5Tg852f886mdwC4M/RO3mj7Bl5WLydXK5w97HhG6+l9jicGPPo1VG3p7KpERESKDQ2uTrJy9wkGf7WZtEw7jUMCmdmvBaX9vXg76m2+2PUFAH3q92Fk85EKFigK4rfD7AcdwQKBVaH3fAULiIiIFDINrk7w9aYjjP0zWKBDnfJMedQRLDBm7RiWHlwKOIIF+jXs59xCxeHgWpj3KKQnOoIFes+HgGBnVyUiIlLsaHAtRH8NFujeNIQJ3W8g3ZbC4BUj2Bi3UcECRYwlZiEsfNoRLFC9LTwyB/yCnF2WiIhIsaTBtZDY7IZXf9zJFxsOA1cOFpjcYTJtqrRxcrUCEHpyOdYtswED9e6FbjPAy9fZZYmIiBRbGlwLQVqmjbCvt7Jke/zfBgt83PFjGpRr4OxynScrHaJmwLkjzq4Ea1IcjY5+71hQsICIiEiRoMG1gCWmZfLE55uIOOgIFni3R2PuaRTMzlM7GbzCESwQUjKET27/hGoB1ZxdrvOknYd5veDQWmdXAsDF2+Fs7cdi7TBawQIiIiJFgAbXAnSlYIH1x9YzYvUIBQtclBTvuGP/xHbwLgnNB4CTH/9ls9mJPGGlebuRWDW0ioiIFAkaXAvI/pMXGPjF5suCBX7c/yMvrX9JwQIXnfoDZj8A52LBvwL0+gaCb3R2VdgzM0lYssTZZYiIiMglNLgWgENJ8PL0SM6l/i9YIKS0H+E7wpkUPQlQsAAARzc5UqhSTkOZmtB7AZQJdXZVIiIiUkRpcM1nK/ec5KNdVjLtmTSuGsTMvs0dwQKb3ubLXV8CChYAYN9y+LoPZKZAcBN49BsoWd7ZVYmIiEgRpsE1H+0/mczgOVux2S20v74cUx9rhqfVzphfx7D0kCNY4Nnmz9K3QV8nV+pkW+fAwqFgbFCrIzz8BfiUdHZVIiIiUsRpcM1HtcqX5KmbQ9m06w+m9roRO2kMXjGciLgIPC2evN7ude6peY+zy3QeY2Dde7DiVcdyox5w/xSn34glIiIirsElvqueMmUKNWrUwNfXl1atWhEZGenskq5oeMdaPFrLzvmMMwz4aQARcRGU8CzBlI5TivfQarfDsjH/G1rbDoOu0zS0ioiISJ4V+cH1v//9L2FhYbz88sts3ryZxo0b07lzZxISEpxdWq4sFgun7afo/3N/Ys7EUMa3DDO7zCzeaVhZ6TB/AERMcyx3Hg+3vwYeRf63n4iIiBQhRf5SgXfffZcnnniC/v37AzBt2jQWL17MzJkzGTNmjJOru9yuzZ8yI2kmyWRQ1acM067vR7UT++HEfmeX5jyRnzqCBTy84IFpcMODzq5IREREXFCRHlwzMjKIjo5m7Nix2es8PDzo1KkTGzZsyPU96enppKenZy8nJiYCkJmZSWZmZoHWeyjxEE/smkqqh4V66Rl8fPh3yu0eXqDHdBXGuyS2B7/AhN4CBdyH/HDx90pB/56RgqU+ugf10T2oj66vIHuY130W6cH11KlT2Gw2KlasmGN9xYoV2b17d67vGT9+PK+++upl63/++WdKlChRIHVeZIyhQ7oXJ602XkssicWvFKcL9IiuIdPqz+7K3TgfkwwxrvVQ/+XLlzu7BMkH6qN7UB/dg/ro+gqihykpKXnarkgPrv/G2LFjCQsLy15OTEykatWq3HHHHQQEBBT48W9Lv43lvyynbK878fLSjUcXtXV2AVcpMzOT5cuXc/vtt6uPLkx9dA/qo3tQH11fQfbw4jfk/6RID67lypXDarVy4sSJHOtPnDhBpUqVcn2Pj48PPj4+l6338vIqtD8onhbPQj2eFBz10T2oj+5BfXQP6qPrK4ge5nV/Rfq2bm9vb5o1a8aKFSuy19ntdlasWEHr1q2dWJmIiIiIFLYifcYVICwsjL59+9K8eXNatmzJ5MmTuXDhQvZTBkRERESkeCjyg2uPHj04efIkL730EvHx8dx4440sW7bsshu2RERERMS9FfnBFWDo0KEMHTrU2WWIiIiIiBMV6WtcRUREREQu0uAqIiIiIi5Bg6uIiIiIuAQNriIiIiLiEjS4ioiIiIhL0OAqIiIiIi5Bg6uIiIiIuAQNriIiIiLiEjS4ioiIiIhL0OAqIiIiIi5Bg6uIiIiIuAQNriIiIiLiEjS4ioiIiIhL8HR2AQXNGANAYmJioRwvMzOTlJQUEhMT8fLyKpRjSv5TH92D+uge1Ef3oD66voLs4cU57eLcdiVuP7gmJSUBULVqVSdXIiIiIiJ/JykpicDAwCu+bjH/NNq6OLvdzvHjxylVqhQWi6XAj5eYmEjVqlU5cuQIAQEBBX48KRjqo3tQH92D+uge1EfXV5A9NMaQlJREcHAwHh5XvpLV7c+4enh4EBISUujHDQgI0B9MN6A+ugf10T2oj+5BfXR9BdXDvzvTepFuzhIRERERl6DBVURERERcggbXfObj48PLL7+Mj4+Ps0uRa6A+ugf10T2oj+5BfXR9RaGHbn9zloiIiIi4B51xFRERERGXoMFVRERERFyCBlcRERERcQkaXEVERETEJWhwzWdTpkyhRo0a+Pr60qpVKyIjI51dklzB+PHjadGiBaVKlaJChQp07dqVPXv25NgmLS2NIUOGULZsWUqWLEn37t05ceKEkyqWvJgwYQIWi4Xhw4dnr1MfXcOxY8fo3bs3ZcuWxc/PjxtuuIFNmzZlv26M4aWXXqJy5cr4+fnRqVMn9u3b58SK5a9sNhvjxo0jNDQUPz8/atWqxeuvv54jf159LHp+/fVX7r33XoKDg7FYLHz//fc5Xs9Lz86cOUOvXr0ICAggKCiIgQMHkpycnO+1anDNR//9738JCwvj5ZdfZvPmzTRu3JjOnTuTkJDg7NIkF2vWrGHIkCFs3LiR5cuXk5mZyR133MGFCxeytxkxYgQ//vgj33zzDWvWrOH48eN069bNiVXL34mKiuKTTz6hUaNGOdarj0Xf2bNnadu2LV5eXixdupRdu3YxadIkSpcunb3NW2+9xQcffMC0adOIiIjA39+fzp07k5aW5sTK5VITJ05k6tSpfPTRR8TExDBx4kTeeustPvzww+xt1Mei58KFCzRu3JgpU6bk+npeetarVy927tzJ8uXLWbRoEb/++itPPvlk/hdrJN+0bNnSDBkyJHvZZrOZ4OBgM378eCdWJXmVkJBgALNmzRpjjDHnzp0zXl5e5ptvvsneJiYmxgBmw4YNzipTriApKclcf/31Zvny5aZ9+/Zm2LBhxhj10VWMHj3atGvX7oqv2+12U6lSJfP2229nrzt37pzx8fExc+fOLYwSJQ/uvvtuM2DAgBzrunXrZnr16mWMUR9dAWC+++677OW89GzXrl0GMFFRUdnbLF261FgsFnPs2LF8rU9nXPNJRkYG0dHRdOrUKXudh4cHnTp1YsOGDU6sTPLq/PnzAJQpUwaA6OhoMjMzc/S0bt26VKtWTT0tgoYMGcLdd9+do1+gPrqKH374gebNm/PQQw9RoUIFmjRpwvTp07NfP3jwIPHx8Tn6GBgYSKtWrdTHIqRNmzasWLGCvXv3ArBt2zbWrVvHnXfeCaiPrigvPduwYQNBQUE0b948e5tOnTrh4eFBREREvtbjma97K8ZOnTqFzWajYsWKOdZXrFiR3bt3O6kqySu73c7w4cNp27YtDRs2BCA+Ph5vb2+CgoJybFuxYkXi4+OdUKVcybx589i8eTNRUVGXvaY+uoYDBw4wdepUwsLCeP7554mKiuKZZ57B29ubvn37Zvcqt79j1ceiY8yYMSQmJlK3bl2sVis2m4033niDXr16AaiPLigvPYuPj6dChQo5Xvf09KRMmTL53lcNriI4ztbt2LGDdevWObsUuUpHjhxh2LBhLF++HF9fX2eXI/+S3W6nefPmvPnmmwA0adKEHTt2MG3aNPr27evk6iSvvv76a7766ivmzJlDgwYN2Lp1K8OHDyc4OFh9lHyhSwXySbly5bBarZfdqXzixAkqVarkpKokL4YOHcqiRYtYtWoVISEh2esrVapERkYG586dy7G9elq0REdHk5CQQNOmTfH09MTT05M1a9bwwQcf4OnpScWKFdVHF1C5cmXq16+fY129evWIjY0FyO6V/o4t2p577jnGjBnDI488wg033MBjjz3GiBEjGD9+PKA+uqK89KxSpUqX3YielZXFmTNn8r2vGlzzibe3N82aNWPFihXZ6+x2OytWrKB169ZOrEyuxBjD0KFD+e6771i5ciWhoaE5Xm/WrBleXl45erpnzx5iY2PV0yKkY8eObN++na1bt2b/at68Ob169cr+b/Wx6Gvbtu1lj6Pbu3cv1atXByA0NJRKlSrl6GNiYiIRERHqYxGSkpKCh0fO0cJqtWK32wH10RXlpWetW7fm3LlzREdHZ2+zcuVK7HY7rVq1yt+C8vVWr2Ju3rx5xsfHx4SHh5tdu3aZJ5980gQFBZn4+Hhnlya5ePrpp01gYKBZvXq1iYuLy/6VkpKSvc2gQYNMtWrVzMqVK82mTZtM69atTevWrZ1YteTFpU8VMEZ9dAWRkZHG09PTvPHGG2bfvn3mq6++MiVKlDCzZ8/O3mbChAkmKCjILFy40Pz+++/m/vvvN6GhoSY1NdWJlcul+vbta6pUqWIWLVpkDh48aBYsWGDKlStnRo0alb2N+lj0JCUlmS1btpgtW7YYwLz77rtmy5Yt5vDhw8aYvPWsS5cupkmTJiYiIsKsW7fOXH/99aZnz575XqsG13z24YcfmmrVqhlvb2/TsmVLs3HjRmeXJFcA5Ppr1qxZ2dukpqaawYMHm9KlS5sSJUqYBx54wMTFxTmvaMmTvw6u6qNr+PHHH03Dhg2Nj4+PqVu3rvn0009zvG632824ceNMxYoVjY+Pj+nYsaPZs2ePk6qV3CQmJpphw4aZatWqGV9fX1OzZk3zwgsvmPT09Oxt1MeiZ9WqVbn+POzbt68xJm89O336tOnZs6cpWbKkCQgIMP379zdJSUn5XqvFmEviLEREREREiihd4yoiIiIiLkGDq4iIiIi4BA2uIiIiIuISNLiKiIiIiEvQ4CoiIiIiLkGDq4iIiIi4BA2uIiIiIuISNLiKiIiIiEvQ4CoikgcdOnRg+PDhzi4jmzGGJ598kjJlymCxWNi6detl24SHhxMUFFTotf2Tfv360bVrV2eXISIuSIOriIgLWrZsGeHh4SxatIi4uDgaNmx42TY9evRg79692cuvvPIKN954Y6HVeOjQoVyH6vfff5/w8PBCq0NE3IenswsQESmubDYbFosFD4+rP4ewf/9+KleuTJs2ba64jZ+fH35+ftdSYq4yMjLw9vb+1+8PDAzMx2pEpDjRGVcRcRkdOnTgmWeeYdSoUZQpU4ZKlSrxyiuvZL+e2xm+c+fOYbFYWL16NQCrV6/GYrHw008/0aRJE/z8/LjttttISEhg6dKl1KtXj4CAAB599FFSUlJyHD8rK4uhQ4cSGBhIuXLlGDduHMaY7NfT09N59tlnqVKlCv7+/rRq1Sr7uPC/r+5/+OEH6tevj4+PD7Gxsbl+1jVr1tCyZUt8fHyoXLkyY8aMISsrC3B81f6f//yH2NhYLBYLNWrUyHUfl14qEB4ezquvvsq2bduwWCxYLJbss57nzp3j8ccfp3z58gQEBHDbbbexbdu27P1cPFM7Y8YMQkND8fX1BRxnfdu1a0dQUBBly5blnnvuYf/+/dnvCw0NBaBJkyZYLBY6dOiQXf+llwqkp6fzzDPPUKFCBXx9fWnXrh1RUVHZr1/s2YoVK2jevDklSpSgTZs27NmzJ9fPLSLuS4OriLiUzz//HH9/fyIiInjrrbd47bXXWL58+VXv55VXXuGjjz7it99+48iRIzz88MNMnjyZOXPmsHjxYn7++Wc+/PDDy47t6elJZGQk77//Pu+++y4zZszIfn3o0KFs2LCBefPm8fvvv/PQQw/RpUsX9u3bl71NSkoKEydOZMaMGezcuZMKFSpcVtuxY8e46667aNGiBdu2bWPq1Kl89tln/N///R/g+Kr9tddeIyQkhLi4uBxD3pX06NGDkSNH0qBBA+Li4oiLi6NHjx4APPTQQ9mDe3R0NE2bNqVjx46cOXMm+/1//PEH8+fPZ8GCBdn/MLhw4QJhYWFs2rSJFStW4OHhwQMPPIDdbgcgMjISgF9++YW4uDgWLFiQa22jRo1i/vz5fP7552zevJnrrruOzp075zg+wAsvvMCkSZPYtGkTnp6eDBgw4B8/t4i4GSMi4iLat29v2rVrl2NdixYtzOjRo40xxhw8eNAAZsuWLdmvnz171gBm1apVxhhjVq1aZQDzyy+/ZG8zfvx4A5j9+/dnr3vqqadM586dcxy7Xr16xm63Z68bPXq0qVevnjHGmMOHDxur1WqOHTuWo76OHTuasWPHGmOMmTVrlgHM1q1b//ZzPv/886ZOnTo5jjVlyhRTsmRJY7PZjDHGvPfee6Z69ep/u59Zs2aZwMDA7OWXX37ZNG7cOMc2a9euNQEBASYtLS3H+lq1aplPPvkk+31eXl4mISHhb4938uRJA5jt27cbY3LvhzHG9O3b19x///3GGGOSk5ONl5eX+eqrr7Jfz8jIMMHBweatt94yxuTes8WLFxvApKam/m1NIuJedMZVRFxKo0aNcixXrlyZhISEa9pPxYoVKVGiBDVr1syx7q/7vemmm7BYLNnLrVu3Zt++fdhsNrZv347NZqN27dqULFky+9eaNWtyfH3u7e192Wf4q5iYGFq3bp3jWG3btiU5OZmjR49e9Wf9O9u2bSM5OZmyZcvmqPvgwYM56q5evTrly5fP8d59+/bRs2dPatasSUBAQPYlC1e6/CE3+/fvJzMzk7Zt22av8/LyomXLlsTExOTY9tL/b5UrVwb4V70XEdelm7NExKV4eXnlWLZYLNlfTV+8yclcct1pZmbmP+7HYrH87X7zIjk5GavVSnR0NFarNcdrJUuWzP5vPz+/HAOpsyUnJ1O5cuUc1+JedOmjtPz9/S97/d5776V69epMnz6d4OBg7HY7DRs2JCMjo0Bq/WvPgKvqkYi4Pg2uIuI2Lp4RjIuLo0mTJgC5Pt/034qIiMixvHHjRq6//nqsVitNmjTBZrORkJDAzTfffE3HqVevHvPnz8cYkz2grV+/nlKlShESEvKv9+vt7Y3NZsuxrmnTpsTHx+Pp6XnFm7xyc/r0afbs2cP06dOzP++6desuOx5w2TEvVatWLby9vVm/fj3Vq1cHHP/YiIqKKlLPzRWRokGXCoiI2/Dz8+Omm25iwoQJxMTEsGbNGl588cV8239sbCxhYWHs2bOHuXPn8uGHHzJs2DAAateuTa9evejTpw8LFizg4MGDREZGMn78eBYvXnxVxxk8eDBHjhzhP//5D7t372bhwoW8/PLLhIWF/atHZ11Uo0YNDh48yNatWzl16hTp6el06tSJ1q1b07VrV37++WcOHTrEb7/9xgsvvMCmTZuuuK/SpUtTtmxZPv30U/744w9WrlxJWFhYjm0qVKiAn58fy5Yt48SJE5w/f/6y/fj7+/P000/z3HPPsWzZMnbt2sUTTzxBSkoKAwcO/NefVUTckwZXEXErM2fOJCsri2bNmjF8+PDsO/HzQ58+fUhNTaVly5YMGTKEYcOG8eSTT2a/PmvWLPr06cPIkSOpU6cOXbt2JSoqimrVql3VcapUqcKSJUuIjIykcePGDBo0iIEDB17zEN69e3e6dOnCrbfeSvny5Zk7dy4Wi4UlS5Zwyy230L9/f2rXrs0jjzzC4cOHqVix4hX35eHhwbx584iOjqZhw4aMGDGCt99+O8c2np6efPDBB3zyyScEBwdz//3357qvCRMm0L17dx577DGaNm3KH3/8wU8//UTp0qWv6fOKiPuxmEsvBhMRERERKaJ0xlVEREREXIIGVxERERFxCRpcRURERMQlaHAVEREREZegwVVEREREXIIGVxERERFxCRpcRURERMQlaHAVEREREZegwVVEREREXIIGVxERERFxCRpcRURERMQl/D8v/wD5Pa6JBAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "census\n", - "Running Isolation Forest\n", - "Running AAD Isolation Forest\n", - "Running Pine Forest\n" -======= - "Running Isolation Forest\n" ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - ] - } - ], - "source": [ - "for dataset in DevNetDataset.avialble_datasets:\n", - " print(dataset)\n", -<<<<<<< HEAD - " %time compare = Compare(DevNetDataset(dataset), n_jobs=15).run().plot(dataset, savefig=True)\n", - " plt.show()" - ] -======= - " %time compare = Compare(DevNetDataset(dataset), n_jobs=15).run().plot(dataset)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dbebe1bb-3b54-4d8f-8624-def1ad6e898e", - "metadata": {}, - "outputs": [], - "source": [] ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", -<<<<<<< HEAD - "version": "3.12.3" -======= - "version": "3.11.2" ->>>>>>> aa93b370fc27725e7768ed3351e2e6bd8307bf92 - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/notebooks/devnet_datasets.ipynb b/docs/notebooks/devnet_datasets.ipynb index b0a4873..9664fd8 100644 --- a/docs/notebooks/devnet_datasets.ipynb +++ b/docs/notebooks/devnet_datasets.ipynb @@ -6,21 +6,54 @@ "id": "ea4ae65a-d555-4b54-96f9-11eed006adc2", "metadata": {}, "outputs": [], - "source": "# %pip install coniferest" + "source": [ + "# %pip install coniferest matplotlib pandas tqdm" + ] }, { "cell_type": "code", + "execution_count": 2, "id": "3d9577061e9494ed", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, "ExecuteTime": { "end_time": "2025-07-03T15:29:14.711984Z", "start_time": "2025-07-03T15:29:13.632289Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false } }, + "outputs": [ + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 7\u001b[39m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtqdm\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m tqdm\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mconiferest\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01maadforest\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AADForest\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mconiferest\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdatasets\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Dataset, DevNetDataset\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mconiferest\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01misoforest\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m IsolationForest\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/coniferest/src/coniferest/aadforest.py:8\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mscipy\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01moptimize\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m minimize\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcalc_trees\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m calc_paths_sum, calc_paths_sum_transpose \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mconiferest\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Coniferest, ConiferestEvaluator\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mlabel\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Label\n\u001b[32m 11\u001b[39m __all__ = [\u001b[33m\"\u001b[39m\u001b[33mAADForest\u001b[39m\u001b[33m\"\u001b[39m]\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/coniferest/src/coniferest/coniferest.py:5\u001b[39m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mwarnings\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m warn\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msklearn\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mensemble\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_bagging\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _generate_indices \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[32m 6\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msklearn\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtree\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_criterion\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MSE \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msklearn\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtree\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_splitter\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m RandomSplitter \u001b[38;5;66;03m# noqa\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/__init__.py:73\u001b[39m\n\u001b[32m 62\u001b[39m \u001b[38;5;66;03m# `_distributor_init` allows distributors to run custom init code.\u001b[39;00m\n\u001b[32m 63\u001b[39m \u001b[38;5;66;03m# For instance, for the Windows wheel, this is used to pre-load the\u001b[39;00m\n\u001b[32m 64\u001b[39m \u001b[38;5;66;03m# vcomp shared library runtime for OpenMP embedded in the sklearn/.libs\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 67\u001b[39m \u001b[38;5;66;03m# later is linked to the OpenMP runtime to make it possible to introspect\u001b[39;00m\n\u001b[32m 68\u001b[39m \u001b[38;5;66;03m# it and importing it first would fail if the OpenMP dll cannot be found.\u001b[39;00m\n\u001b[32m 69\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ( \u001b[38;5;66;03m# noqa: F401 E402\u001b[39;00m\n\u001b[32m 70\u001b[39m __check_build,\n\u001b[32m 71\u001b[39m _distributor_init,\n\u001b[32m 72\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbase\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m clone \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n\u001b[32m 74\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_show_versions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m show_versions \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n\u001b[32m 76\u001b[39m _submodules = [\n\u001b[32m 77\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcalibration\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 78\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcluster\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 114\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcompose\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 115\u001b[39m ]\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/base.py:19\u001b[39m\n\u001b[32m 17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_config\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexceptions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m InconsistentVersionWarning\n\u001b[32m---> \u001b[39m\u001b[32m19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_metadata_requests\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _MetadataRequester, _routing_enabled\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_missing\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m is_scalar_nan\n\u001b[32m 21\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_param_validation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m validate_parameter_constraints\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/__init__.py:9\u001b[39m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m metadata_routing\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_bunch\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Bunch\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_chunking\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m gen_batches, gen_even_slices\n\u001b[32m 11\u001b[39m \u001b[38;5;66;03m# Make _safe_indexing importable from here for backward compat as this particular\u001b[39;00m\n\u001b[32m 12\u001b[39m \u001b[38;5;66;03m# helper is considered semi-private and typically very useful for third-party\u001b[39;00m\n\u001b[32m 13\u001b[39m \u001b[38;5;66;03m# libraries that want to comply with scikit-learn's estimator API. In particular,\u001b[39;00m\n\u001b[32m 14\u001b[39m \u001b[38;5;66;03m# _safe_indexing was included in our public API documentation despite the leading\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;66;03m# `_` in its name.\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_indexing\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 17\u001b[39m _safe_indexing, \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[32m 18\u001b[39m resample,\n\u001b[32m 19\u001b[39m shuffle,\n\u001b[32m 20\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/_chunking.py:11\u001b[39m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m 10\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_config\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_config\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_param_validation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Interval, validate_params\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mchunk_generator\u001b[39m(gen, chunksize):\n\u001b[32m 15\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Chunk generator, ``gen`` into lists of length ``chunksize``. The last\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[33;03m chunk may have a length less than ``chunksize``.\"\"\"\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/_param_validation.py:17\u001b[39m\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mscipy\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01msparse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m csr_matrix, issparse\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_config\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[32m---> \u001b[39m\u001b[32m17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mvalidation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _is_arraylike_not_scalar\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mInvalidParameterError\u001b[39;00m(\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m):\n\u001b[32m 21\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Custom exception to be raised when the parameter of a class/method/function\u001b[39;00m\n\u001b[32m 22\u001b[39m \u001b[33;03m does not have a valid type or value.\u001b[39;00m\n\u001b[32m 23\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/validation.py:21\u001b[39m\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_config \u001b[38;5;28;01mas\u001b[39;00m _get_config\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexceptions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DataConversionWarning, NotFittedError, PositiveSpectrumWarning\n\u001b[32m---> \u001b[39m\u001b[32m21\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_array_api\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _asarray_with_order, _is_numpy_namespace, get_namespace\n\u001b[32m 22\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdeprecation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _deprecate_force_all_finite\n\u001b[32m 23\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mfixes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ComplexWarning, _preserve_dia_indices_dtype\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/_array_api.py:20\u001b[39m\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexternals\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m array_api_extra \u001b[38;5;28;01mas\u001b[39;00m xpx\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexternals\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01marray_api_compat\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m numpy \u001b[38;5;28;01mas\u001b[39;00m np_compat\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mfixes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m parse_version\n\u001b[32m 22\u001b[39m \u001b[38;5;66;03m# TODO: complete __all__\u001b[39;00m\n\u001b[32m 23\u001b[39m __all__ = [\u001b[33m\"\u001b[39m\u001b[33mxpx\u001b[39m\u001b[33m\"\u001b[39m] \u001b[38;5;66;03m# we import xpx here just to re-export it, need this to appease ruff\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/fixes.py:20\u001b[39m\n\u001b[32m 17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mscipy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m optimize\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpd\u001b[39;00m\n\u001b[32m 21\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[32m 22\u001b[39m pd = \u001b[38;5;28;01mNone\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/__init__.py:49\u001b[39m\n\u001b[32m 46\u001b[39m \u001b[38;5;66;03m# let init-time option registration happen\u001b[39;00m\n\u001b[32m 47\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mconfig_init\u001b[39;00m \u001b[38;5;66;03m# pyright: ignore[reportUnusedImport] # noqa: F401\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m49\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 50\u001b[39m \u001b[38;5;66;03m# dtype\u001b[39;00m\n\u001b[32m 51\u001b[39m ArrowDtype,\n\u001b[32m 52\u001b[39m Int8Dtype,\n\u001b[32m 53\u001b[39m Int16Dtype,\n\u001b[32m 54\u001b[39m Int32Dtype,\n\u001b[32m 55\u001b[39m Int64Dtype,\n\u001b[32m 56\u001b[39m UInt8Dtype,\n\u001b[32m 57\u001b[39m UInt16Dtype,\n\u001b[32m 58\u001b[39m UInt32Dtype,\n\u001b[32m 59\u001b[39m UInt64Dtype,\n\u001b[32m 60\u001b[39m Float32Dtype,\n\u001b[32m 61\u001b[39m Float64Dtype,\n\u001b[32m 62\u001b[39m CategoricalDtype,\n\u001b[32m 63\u001b[39m PeriodDtype,\n\u001b[32m 64\u001b[39m IntervalDtype,\n\u001b[32m 65\u001b[39m DatetimeTZDtype,\n\u001b[32m 66\u001b[39m StringDtype,\n\u001b[32m 67\u001b[39m BooleanDtype,\n\u001b[32m 68\u001b[39m \u001b[38;5;66;03m# missing\u001b[39;00m\n\u001b[32m 69\u001b[39m NA,\n\u001b[32m 70\u001b[39m isna,\n\u001b[32m 71\u001b[39m isnull,\n\u001b[32m 72\u001b[39m notna,\n\u001b[32m 73\u001b[39m notnull,\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# indexes\u001b[39;00m\n\u001b[32m 75\u001b[39m Index,\n\u001b[32m 76\u001b[39m CategoricalIndex,\n\u001b[32m 77\u001b[39m RangeIndex,\n\u001b[32m 78\u001b[39m MultiIndex,\n\u001b[32m 79\u001b[39m IntervalIndex,\n\u001b[32m 80\u001b[39m TimedeltaIndex,\n\u001b[32m 81\u001b[39m DatetimeIndex,\n\u001b[32m 82\u001b[39m PeriodIndex,\n\u001b[32m 83\u001b[39m IndexSlice,\n\u001b[32m 84\u001b[39m \u001b[38;5;66;03m# tseries\u001b[39;00m\n\u001b[32m 85\u001b[39m NaT,\n\u001b[32m 86\u001b[39m Period,\n\u001b[32m 87\u001b[39m period_range,\n\u001b[32m 88\u001b[39m Timedelta,\n\u001b[32m 89\u001b[39m timedelta_range,\n\u001b[32m 90\u001b[39m Timestamp,\n\u001b[32m 91\u001b[39m date_range,\n\u001b[32m 92\u001b[39m bdate_range,\n\u001b[32m 93\u001b[39m Interval,\n\u001b[32m 94\u001b[39m interval_range,\n\u001b[32m 95\u001b[39m DateOffset,\n\u001b[32m 96\u001b[39m \u001b[38;5;66;03m# conversion\u001b[39;00m\n\u001b[32m 97\u001b[39m to_numeric,\n\u001b[32m 98\u001b[39m to_datetime,\n\u001b[32m 99\u001b[39m to_timedelta,\n\u001b[32m 100\u001b[39m \u001b[38;5;66;03m# misc\u001b[39;00m\n\u001b[32m 101\u001b[39m Flags,\n\u001b[32m 102\u001b[39m Grouper,\n\u001b[32m 103\u001b[39m factorize,\n\u001b[32m 104\u001b[39m unique,\n\u001b[32m 105\u001b[39m value_counts,\n\u001b[32m 106\u001b[39m NamedAgg,\n\u001b[32m 107\u001b[39m array,\n\u001b[32m 108\u001b[39m Categorical,\n\u001b[32m 109\u001b[39m set_eng_float_format,\n\u001b[32m 110\u001b[39m Series,\n\u001b[32m 111\u001b[39m DataFrame,\n\u001b[32m 112\u001b[39m )\n\u001b[32m 114\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdtypes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdtypes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m SparseDtype\n\u001b[32m 116\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtseries\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m infer_freq\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/api.py:47\u001b[39m\n\u001b[32m 45\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mconstruction\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m array\n\u001b[32m 46\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mflags\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Flags\n\u001b[32m---> \u001b[39m\u001b[32m47\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 48\u001b[39m Grouper,\n\u001b[32m 49\u001b[39m NamedAgg,\n\u001b[32m 50\u001b[39m )\n\u001b[32m 51\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mindexes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 52\u001b[39m CategoricalIndex,\n\u001b[32m 53\u001b[39m DatetimeIndex,\n\u001b[32m (...)\u001b[39m\u001b[32m 59\u001b[39m TimedeltaIndex,\n\u001b[32m 60\u001b[39m )\n\u001b[32m 61\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mindexes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdatetimes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 62\u001b[39m bdate_range,\n\u001b[32m 63\u001b[39m date_range,\n\u001b[32m 64\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/groupby/__init__.py:1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgeneric\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 2\u001b[39m DataFrameGroupBy,\n\u001b[32m 3\u001b[39m NamedAgg,\n\u001b[32m 4\u001b[39m SeriesGroupBy,\n\u001b[32m 5\u001b[39m )\n\u001b[32m 6\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m GroupBy\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgrouper\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Grouper\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/groupby/generic.py:1329\u001b[39m\n\u001b[32m 1325\u001b[39m result = \u001b[38;5;28mself\u001b[39m._op_via_apply(\u001b[33m\"\u001b[39m\u001b[33munique\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 1326\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[32m-> \u001b[39m\u001b[32m1329\u001b[39m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[38;5;250;43m \u001b[39;49m\u001b[34;43;01mDataFrameGroupBy\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43mGroupBy\u001b[49m\u001b[43m[\u001b[49m\u001b[43mDataFrame\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 1330\u001b[39m \u001b[43m \u001b[49m\u001b[43m_agg_examples_doc\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mdedent\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1331\u001b[39m \u001b[38;5;250;43m \u001b[39;49m\u001b[33;43;03m\"\"\"\u001b[39;49;00m\n\u001b[32m 1332\u001b[39m \u001b[33;43;03m Examples\u001b[39;49;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 1417\u001b[39m \u001b[33;43;03m \"\"\"\u001b[39;49;00m\n\u001b[32m 1418\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1420\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;129;43m@doc\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m_agg_template_frame\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexamples\u001b[49m\u001b[43m=\u001b[49m\u001b[43m_agg_examples_doc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mklass\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mDataFrame\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 1421\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mdef\u001b[39;49;00m\u001b[38;5;250;43m \u001b[39;49m\u001b[34;43maggregate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/groupby/generic.py:1758\u001b[39m, in \u001b[36mDataFrameGroupBy\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 1755\u001b[39m concatenated = concatenated.reindex(concat_index, axis=other_axis, copy=\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[32m 1756\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._set_result_index_ordered(concatenated)\n\u001b[32m-> \u001b[39m\u001b[32m1758\u001b[39m __examples_dataframe_doc = \u001b[43mdedent\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1759\u001b[39m \u001b[38;5;250;43m \u001b[39;49m\u001b[33;43;03m\"\"\"\u001b[39;49;00m\n\u001b[32m 1760\u001b[39m \u001b[33;43;03m>>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',\u001b[39;49;00m\n\u001b[32m 1761\u001b[39m \u001b[33;43;03m... 'foo', 'bar'],\u001b[39;49;00m\n\u001b[32m 1762\u001b[39m \u001b[33;43;03m... 'B' : ['one', 'one', 'two', 'three',\u001b[39;49;00m\n\u001b[32m 1763\u001b[39m \u001b[33;43;03m... 'two', 'two'],\u001b[39;49;00m\n\u001b[32m 1764\u001b[39m \u001b[33;43;03m... 'C' : [1, 5, 5, 2, 5, 5],\u001b[39;49;00m\n\u001b[32m 1765\u001b[39m \u001b[33;43;03m... 'D' : [2.0, 5., 8., 1., 2., 9.]})\u001b[39;49;00m\n\u001b[32m 1766\u001b[39m \u001b[33;43;03m>>> grouped = df.groupby('A')[['C', 'D']]\u001b[39;49;00m\n\u001b[32m 1767\u001b[39m \u001b[33;43;03m>>> grouped.transform(lambda x: (x - x.mean()) / x.std())\u001b[39;49;00m\n\u001b[32m 1768\u001b[39m \u001b[33;43;03m C D\u001b[39;49;00m\n\u001b[32m 1769\u001b[39m \u001b[33;43;03m0 -1.154701 -0.577350\u001b[39;49;00m\n\u001b[32m 1770\u001b[39m \u001b[33;43;03m1 0.577350 0.000000\u001b[39;49;00m\n\u001b[32m 1771\u001b[39m \u001b[33;43;03m2 0.577350 1.154701\u001b[39;49;00m\n\u001b[32m 1772\u001b[39m \u001b[33;43;03m3 -1.154701 -1.000000\u001b[39;49;00m\n\u001b[32m 1773\u001b[39m \u001b[33;43;03m4 0.577350 -0.577350\u001b[39;49;00m\n\u001b[32m 1774\u001b[39m \u001b[33;43;03m5 0.577350 1.000000\u001b[39;49;00m\n\u001b[32m 1775\u001b[39m \n\u001b[32m 1776\u001b[39m \u001b[33;43;03mBroadcast result of the transformation\u001b[39;49;00m\n\u001b[32m 1777\u001b[39m \n\u001b[32m 1778\u001b[39m \u001b[33;43;03m>>> grouped.transform(lambda x: x.max() - x.min())\u001b[39;49;00m\n\u001b[32m 1779\u001b[39m \u001b[33;43;03m C D\u001b[39;49;00m\n\u001b[32m 1780\u001b[39m \u001b[33;43;03m0 4.0 6.0\u001b[39;49;00m\n\u001b[32m 1781\u001b[39m \u001b[33;43;03m1 3.0 8.0\u001b[39;49;00m\n\u001b[32m 1782\u001b[39m \u001b[33;43;03m2 4.0 6.0\u001b[39;49;00m\n\u001b[32m 1783\u001b[39m \u001b[33;43;03m3 3.0 8.0\u001b[39;49;00m\n\u001b[32m 1784\u001b[39m \u001b[33;43;03m4 4.0 6.0\u001b[39;49;00m\n\u001b[32m 1785\u001b[39m \u001b[33;43;03m5 3.0 8.0\u001b[39;49;00m\n\u001b[32m 1786\u001b[39m \n\u001b[32m 1787\u001b[39m \u001b[33;43;03m>>> grouped.transform(\"mean\")\u001b[39;49;00m\n\u001b[32m 1788\u001b[39m \u001b[33;43;03m C D\u001b[39;49;00m\n\u001b[32m 1789\u001b[39m \u001b[33;43;03m0 3.666667 4.0\u001b[39;49;00m\n\u001b[32m 1790\u001b[39m \u001b[33;43;03m1 4.000000 5.0\u001b[39;49;00m\n\u001b[32m 1791\u001b[39m \u001b[33;43;03m2 3.666667 4.0\u001b[39;49;00m\n\u001b[32m 1792\u001b[39m \u001b[33;43;03m3 4.000000 5.0\u001b[39;49;00m\n\u001b[32m 1793\u001b[39m \u001b[33;43;03m4 3.666667 4.0\u001b[39;49;00m\n\u001b[32m 1794\u001b[39m \u001b[33;43;03m5 4.000000 5.0\u001b[39;49;00m\n\u001b[32m 1795\u001b[39m \n\u001b[32m 1796\u001b[39m \u001b[33;43;03m.. versionchanged:: 1.3.0\u001b[39;49;00m\n\u001b[32m 1797\u001b[39m \n\u001b[32m 1798\u001b[39m \u001b[33;43;03mThe resulting dtype will reflect the return value of the passed ``func``,\u001b[39;49;00m\n\u001b[32m 1799\u001b[39m \u001b[33;43;03mfor example:\u001b[39;49;00m\n\u001b[32m 1800\u001b[39m \n\u001b[32m 1801\u001b[39m \u001b[33;43;03m>>> grouped.transform(lambda x: x.astype(int).max())\u001b[39;49;00m\n\u001b[32m 1802\u001b[39m \u001b[33;43;03mC D\u001b[39;49;00m\n\u001b[32m 1803\u001b[39m \u001b[33;43;03m0 5 8\u001b[39;49;00m\n\u001b[32m 1804\u001b[39m \u001b[33;43;03m1 5 9\u001b[39;49;00m\n\u001b[32m 1805\u001b[39m \u001b[33;43;03m2 5 8\u001b[39;49;00m\n\u001b[32m 1806\u001b[39m \u001b[33;43;03m3 5 9\u001b[39;49;00m\n\u001b[32m 1807\u001b[39m \u001b[33;43;03m4 5 8\u001b[39;49;00m\n\u001b[32m 1808\u001b[39m \u001b[33;43;03m5 5 9\u001b[39;49;00m\n\u001b[32m 1809\u001b[39m \u001b[33;43;03m\"\"\"\u001b[39;49;00m\n\u001b[32m 1810\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1812\u001b[39m \u001b[38;5;129m@Substitution\u001b[39m(klass=\u001b[33m\"\u001b[39m\u001b[33mDataFrame\u001b[39m\u001b[33m\"\u001b[39m, example=__examples_dataframe_doc)\n\u001b[32m 1813\u001b[39m \u001b[38;5;129m@Appender\u001b[39m(_transform_template)\n\u001b[32m 1814\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mtransform\u001b[39m(\u001b[38;5;28mself\u001b[39m, func, *args, engine=\u001b[38;5;28;01mNone\u001b[39;00m, engine_kwargs=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 1815\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._transform(\n\u001b[32m 1816\u001b[39m func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs\n\u001b[32m 1817\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.13.1-linux-x86_64-gnu/lib/python3.13/textwrap.py:466\u001b[39m, in \u001b[36mdedent\u001b[39m\u001b[34m(text)\u001b[39m\n\u001b[32m 462\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m line \u001b[38;5;129;01mor\u001b[39;00m line.startswith(margin), \\\n\u001b[32m 463\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mline = \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m, margin = \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m % (line, margin)\n\u001b[32m 465\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m margin:\n\u001b[32m--> \u001b[39m\u001b[32m466\u001b[39m text = \u001b[43mre\u001b[49m\u001b[43m.\u001b[49m\u001b[43msub\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mr\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m(?m)^\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m+\u001b[49m\u001b[43m \u001b[49m\u001b[43mmargin\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 467\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m text\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.13.1-linux-x86_64-gnu/lib/python3.13/re/__init__.py:208\u001b[39m, in \u001b[36msub\u001b[39m\u001b[34m(pattern, repl, string, count, flags, *args)\u001b[39m\n\u001b[32m 202\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mwarnings\u001b[39;00m\n\u001b[32m 203\u001b[39m warnings.warn(\n\u001b[32m 204\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m'\u001b[39m\u001b[33mcount\u001b[39m\u001b[33m'\u001b[39m\u001b[33m is passed as positional argument\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 205\u001b[39m \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m, stacklevel=\u001b[32m2\u001b[39m\n\u001b[32m 206\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m208\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_compile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpattern\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43msub\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrepl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstring\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[31mKeyboardInterrupt\u001b[39m: " + ] + } + ], "source": [ "from collections import defaultdict\n", "\n", @@ -34,12 +67,11 @@ "from coniferest.label import Label\n", "from coniferest.pineforest import PineForest\n", "from coniferest.session.oracle import OracleSession, create_oracle_session" - ], - "outputs": [], - "execution_count": 1 + ] }, { "cell_type": "code", + "execution_count": null, "id": "initial_id", "metadata": { "ExecuteTime": { @@ -47,6 +79,7 @@ "start_time": "2025-07-03T15:29:15.707980Z" } }, + "outputs": [], "source": [ "class Compare:\n", " models = {\n", @@ -55,9 +88,10 @@ " 'Pine Forest': PineForest,\n", " }\n", "\n", - " def __init__(self, dataset: Dataset, *, iterations=100, n_jobs=-1):\n", + " def __init__(self, dataset: Dataset, *, iterations=100, n_jobs=-1, sampletrees_per_batch=1<<20):\n", " self.model_kwargs = {\n", " 'n_trees': 128,\n", + " 'sampletrees_per_batch': sampletrees_per_batch,\n", " 'n_jobs': n_jobs,\n", " }\n", " self.session_kwargs = {\n", @@ -78,8 +112,11 @@ " }\n", "\n", " def run(self, random_seeds):\n", + " assert len(random_seeds) == len(set(random_seeds)), \"random seeds must be different\"\n", + " \n", " results = defaultdict(dict)\n", "\n", + " futures = []\n", " for random_seed in tqdm(random_seeds):\n", " sessions = self.get_sessions(random_seed)\n", " for name, session in sessions.items():\n", @@ -109,74 +146,94 @@ " plt.grid()\n", " plt.legend()\n", " if savefig:\n", - " plt.savefig(f'{dataset}.pdf')\n", + " plt.savefig(f'{dataset_name}.pdf')\n", "\n", " return self" - ], + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "929fd77b-3333-4937-90aa-d2804151d868", + "metadata": {}, "outputs": [], - "execution_count": 2 + "source": [ + "import pickle\n", + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "\n", + "class GalaxyZoo2Dataset(Dataset):\n", + " def __init__(self, path: Path, *, anomaly_class='Class6.1', anomaly_threshold=0.9):\n", + " astronomaly = pd.read_parquet(path / \"astronomaly.parquet\")\n", + " self.data = astronomaly.drop(columns=['GalaxyID', 'anomaly']).to_numpy().copy(order='C')\n", + " ids = astronomaly['GalaxyID'].to_numpy()\n", + "\n", + " solutions = pd.read_csv(path / \"training_solutions_rev1.csv\", index_col=\"GalaxyID\")\n", + " anomaly = solutions[anomaly_class][ids] >= anomaly_threshold\n", + " self.labels = np.full(anomaly.shape, Label.R)\n", + " self.labels[anomaly] = Label.A\n", + "\n", + "\n", + "seeds = range(200, 400)\n", + "\n", + "path = Path(\"/home/hombit/gz2\")\n", + "dataset_obj = GalaxyZoo2Dataset(path)\n", + "%time compare_zoo = Compare(dataset_obj, iterations=100, n_jobs=24, sampletrees_per_batch=1<<16).run(seeds)\n", + "compare_zoo.plot(\"Galaxy Zoo 2 (Anything odd? 90%)\", savefig=True)\n", + "with open(\"galaxyzoo2_compare.pickle\", \"wb\") as fh:\n", + " pickle.dump(compare_zoo, fh)" + ] }, { "cell_type": "code", + "execution_count": null, "id": "71c337b3577915d5", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, "ExecuteTime": { "end_time": "2025-07-03T15:35:53.300312Z", "start_time": "2025-07-03T15:34:16.696646Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false } }, + "outputs": [], "source": [ + "%%time\n", + "\n", + "import pickle\n", + "\n", + "from joblib.parallel import delayed, Parallel\n", + "\n", "print(DevNetDataset.avialble_datasets)\n", "\n", "seeds = range(200)\n", + "compare_delayed = delayed(\n", + " lambda dataset: Compare(DevNetDataset(dataset), iterations=100, n_jobs=48, sampletrees_per_batch=1<<16).run(seeds),\n", + ")\n", + "compare_ = Parallel(\n", + " n_jobs=len(DevNetDataset.avialble_datasets),\n", + ")(compare_delayed(dataset) for dataset in DevNetDataset.avialble_datasets)\n", "\n", - "for dataset in DevNetDataset.avialble_datasets:\n", - " print(dataset)\n", - " %time compare = Compare(DevNetDataset(dataset), iterations=100, n_jobs=-1).run(seeds).plot(dataset, savefig=True)\n", - " plt.show()" - ], - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['donors', 'census', 'fraud', 'celeba', 'backdoor', 'campaign', 'thyroid']\n", - "donors\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/200 [01:35 \u001B[39m\u001B[32m7\u001B[39m \u001B[43mget_ipython\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m.\u001B[49m\u001B[43mrun_line_magic\u001B[49m\u001B[43m(\u001B[49m\u001B[33;43m'\u001B[39;49m\u001B[33;43mtime\u001B[39;49m\u001B[33;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[33;43m'\u001B[39;49m\u001B[33;43mcompare = Compare(DevNetDataset(dataset), iterations=100, n_jobs=-1).run(seeds).plot(dataset, savefig=True)\u001B[39;49m\u001B[33;43m'\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[32m 8\u001B[39m plt.show()\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/.virtualenvs/coniferest/lib/python3.12/site-packages/IPython/core/interactiveshell.py:2488\u001B[39m, in \u001B[36mInteractiveShell.run_line_magic\u001B[39m\u001B[34m(self, magic_name, line, _stack_depth)\u001B[39m\n\u001B[32m 2486\u001B[39m kwargs[\u001B[33m'\u001B[39m\u001B[33mlocal_ns\u001B[39m\u001B[33m'\u001B[39m] = \u001B[38;5;28mself\u001B[39m.get_local_scope(stack_depth)\n\u001B[32m 2487\u001B[39m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m.builtin_trap:\n\u001B[32m-> \u001B[39m\u001B[32m2488\u001B[39m result = \u001B[43mfn\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 2490\u001B[39m \u001B[38;5;66;03m# The code below prevents the output from being displayed\u001B[39;00m\n\u001B[32m 2491\u001B[39m \u001B[38;5;66;03m# when using magics with decorator @output_can_be_silenced\u001B[39;00m\n\u001B[32m 2492\u001B[39m \u001B[38;5;66;03m# when the last Python token in the expression is a ';'.\u001B[39;00m\n\u001B[32m 2493\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mgetattr\u001B[39m(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, \u001B[38;5;28;01mFalse\u001B[39;00m):\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/.virtualenvs/coniferest/lib/python3.12/site-packages/IPython/core/magics/execution.py:1390\u001B[39m, in \u001B[36mExecutionMagics.time\u001B[39m\u001B[34m(self, line, cell, local_ns)\u001B[39m\n\u001B[32m 1388\u001B[39m st = clock2()\n\u001B[32m 1389\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m-> \u001B[39m\u001B[32m1390\u001B[39m \u001B[43mexec\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mglob\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlocal_ns\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 1391\u001B[39m out = \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[32m 1392\u001B[39m \u001B[38;5;66;03m# multi-line %%time case\u001B[39;00m\n", - "\u001B[36mFile \u001B[39m\u001B[32m:1\u001B[39m\n", - "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[2]\u001B[39m\u001B[32m, line 36\u001B[39m, in \u001B[36mCompare.run\u001B[39m\u001B[34m(self, random_seeds)\u001B[39m\n\u001B[32m 34\u001B[39m sessions = \u001B[38;5;28mself\u001B[39m.get_sessions(random_seed)\n\u001B[32m 35\u001B[39m \u001B[38;5;28;01mfor\u001B[39;00m name, session \u001B[38;5;129;01min\u001B[39;00m sessions.items():\n\u001B[32m---> \u001B[39m\u001B[32m36\u001B[39m \u001B[43msession\u001B[49m\u001B[43m.\u001B[49m\u001B[43mrun\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 37\u001B[39m anomalies = np.cumsum(np.array(\u001B[38;5;28mlist\u001B[39m(session.known_labels.values())) == Label.A)\n\u001B[32m 38\u001B[39m results[name][random_seed] = anomalies\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/session/__init__.py:158\u001B[39m, in \u001B[36mSession.run\u001B[39m\u001B[34m(self)\u001B[39m\n\u001B[32m 156\u001B[39m known_data = \u001B[38;5;28mself\u001B[39m._data[\u001B[38;5;28mlist\u001B[39m(\u001B[38;5;28mself\u001B[39m._known_labels.keys())]\n\u001B[32m 157\u001B[39m known_labels = np.fromiter(\u001B[38;5;28mself\u001B[39m._known_labels.values(), dtype=\u001B[38;5;28mint\u001B[39m, count=\u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m._known_labels))\n\u001B[32m--> \u001B[39m\u001B[32m158\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m.\u001B[49m\u001B[43mfit_known\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43m_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mknown_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mknown_labels\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 160\u001B[39m \u001B[38;5;28mself\u001B[39m._invoke_callbacks(\u001B[38;5;28mself\u001B[39m._on_refit_cb, \u001B[38;5;28mself\u001B[39m)\n\u001B[32m 162\u001B[39m \u001B[38;5;28mself\u001B[39m._scores = \u001B[38;5;28mself\u001B[39m.model.score_samples(\u001B[38;5;28mself\u001B[39m._data)\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/pineforest.py:196\u001B[39m, in \u001B[36mPineForest.fit_known\u001B[39m\u001B[34m(self, data, known_data, known_labels)\u001B[39m\n\u001B[32m 194\u001B[39m \u001B[38;5;28mself\u001B[39m._expand_trees(data, \u001B[38;5;28mself\u001B[39m.n_trees)\n\u001B[32m 195\u001B[39m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[32m--> \u001B[39m\u001B[32m196\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43m_expand_trees\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mn_trees\u001B[49m\u001B[43m \u001B[49m\u001B[43m+\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mn_spare_trees\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 197\u001B[39m \u001B[38;5;28mself\u001B[39m._contract_trees(known_data, known_labels, \u001B[38;5;28mself\u001B[39m.n_trees)\n\u001B[32m 199\u001B[39m \u001B[38;5;28mself\u001B[39m.evaluator = ConiferestEvaluator(\u001B[38;5;28mself\u001B[39m)\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/pineforest.py:101\u001B[39m, in \u001B[36mPineForest._expand_trees\u001B[39m\u001B[34m(self, data, n_trees)\u001B[39m\n\u001B[32m 99\u001B[39m n = n_trees - \u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m.trees)\n\u001B[32m 100\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m n > \u001B[32m0\u001B[39m:\n\u001B[32m--> \u001B[39m\u001B[32m101\u001B[39m \u001B[38;5;28mself\u001B[39m.trees.extend(\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mbuild_trees\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mn\u001B[49m\u001B[43m)\u001B[49m)\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/coniferest.py:117\u001B[39m, in \u001B[36mConiferest.build_trees\u001B[39m\u001B[34m(self, data, n_trees)\u001B[39m\n\u001B[32m 109\u001B[39m indices = _generate_indices(\n\u001B[32m 110\u001B[39m random_state=random_state,\n\u001B[32m 111\u001B[39m bootstrap=\u001B[38;5;28mself\u001B[39m.bootstrap_samples,\n\u001B[32m 112\u001B[39m n_population=n_population,\n\u001B[32m 113\u001B[39m n_samples=n_samples,\n\u001B[32m 114\u001B[39m )\n\u001B[32m 116\u001B[39m subsamples = data[indices, :]\n\u001B[32m--> \u001B[39m\u001B[32m117\u001B[39m tree = \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mbuild_one_tree\u001B[49m\u001B[43m(\u001B[49m\u001B[43msubsamples\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 118\u001B[39m trees.append(tree)\n\u001B[32m 120\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m trees\n", - "\u001B[36mFile \u001B[39m\u001B[32m~/projects/supernovaAD/coniferest/src/coniferest/coniferest.py:122\u001B[39m, in \u001B[36mConiferest.build_one_tree\u001B[39m\u001B[34m(self, data)\u001B[39m\n\u001B[32m 118\u001B[39m trees.append(tree)\n\u001B[32m 120\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m trees\n\u001B[32m--> \u001B[39m\u001B[32m122\u001B[39m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34mbuild_one_tree\u001B[39m(\u001B[38;5;28mself\u001B[39m, data):\n\u001B[32m 123\u001B[39m \u001B[38;5;250m \u001B[39m\u001B[33;03m\"\"\"\u001B[39;00m\n\u001B[32m 124\u001B[39m \u001B[33;03m Build just one tree.\u001B[39;00m\n\u001B[32m 125\u001B[39m \n\u001B[32m (...)\u001B[39m\u001B[32m 133\u001B[39m \u001B[33;03m A tree.\u001B[39;00m\n\u001B[32m 134\u001B[39m \u001B[33;03m \"\"\"\u001B[39;00m\n\u001B[32m 135\u001B[39m \u001B[38;5;66;03m# Hollow plug\u001B[39;00m\n", - "\u001B[31mKeyboardInterrupt\u001B[39m: " - ] - } - ], - "execution_count": 5 + "for dataset, compare_obj in zip(DevNetDataset.avialble_datasets, compare_):\n", + " print(f\"Plot {dataset}\")\n", + " compare_obj.plot(dataset, savefig=True)\n", + "\n", + "for dataset, compare_obj in zip(DevNetDataset.avialble_datasets, compare_):\n", + " print(f\"Save Compare object for {dataset}\")\n", + " with open(f'{dataset}_compare.pickle', 'wb') as fh:\n", + " pickle.dump(compare_obj, fh)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb3c8e56-306a-4bd7-a756-f8489deb1c22", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -195,7 +252,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.13.1" } }, "nbformat": 4, diff --git a/src/coniferest/datasets/__init__.py b/src/coniferest/datasets/__init__.py index 37e53f4..c33c7ce 100644 --- a/src/coniferest/datasets/__init__.py +++ b/src/coniferest/datasets/__init__.py @@ -158,6 +158,8 @@ def __init__(self, name: str): if name not in self.avialble_datasets: raise ValueError(f"Dataset {name} is not available. Available datasets are: {self.avialble_datasets}") + self.name = name + df = pd.read_csv(self._dataset_urls[name]) # Last column is for class, the rest are features From 111723772e7326f7da7d627b18914b9f115c525e Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Thu, 10 Jul 2025 23:16:45 +0300 Subject: [PATCH 37/40] Update devnet_datasets.ipynb --- docs/notebooks/devnet_datasets.ipynb | 45 ++++++++-------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/docs/notebooks/devnet_datasets.ipynb b/docs/notebooks/devnet_datasets.ipynb index 9664fd8..a202a8b 100644 --- a/docs/notebooks/devnet_datasets.ipynb +++ b/docs/notebooks/devnet_datasets.ipynb @@ -24,36 +24,7 @@ "outputs_hidden": false } }, - "outputs": [ - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 7\u001b[39m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtqdm\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m tqdm\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mconiferest\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01maadforest\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AADForest\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mconiferest\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdatasets\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Dataset, DevNetDataset\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mconiferest\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01misoforest\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m IsolationForest\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/coniferest/src/coniferest/aadforest.py:8\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mscipy\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01moptimize\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m minimize\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcalc_trees\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m calc_paths_sum, calc_paths_sum_transpose \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mconiferest\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Coniferest, ConiferestEvaluator\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mlabel\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Label\n\u001b[32m 11\u001b[39m __all__ = [\u001b[33m\"\u001b[39m\u001b[33mAADForest\u001b[39m\u001b[33m\"\u001b[39m]\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/coniferest/src/coniferest/coniferest.py:5\u001b[39m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mwarnings\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m warn\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msklearn\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mensemble\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_bagging\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _generate_indices \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[32m 6\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msklearn\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtree\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_criterion\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MSE \u001b[38;5;66;03m# noqa\u001b[39;00m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msklearn\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtree\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_splitter\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m RandomSplitter \u001b[38;5;66;03m# noqa\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/__init__.py:73\u001b[39m\n\u001b[32m 62\u001b[39m \u001b[38;5;66;03m# `_distributor_init` allows distributors to run custom init code.\u001b[39;00m\n\u001b[32m 63\u001b[39m \u001b[38;5;66;03m# For instance, for the Windows wheel, this is used to pre-load the\u001b[39;00m\n\u001b[32m 64\u001b[39m \u001b[38;5;66;03m# vcomp shared library runtime for OpenMP embedded in the sklearn/.libs\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 67\u001b[39m \u001b[38;5;66;03m# later is linked to the OpenMP runtime to make it possible to introspect\u001b[39;00m\n\u001b[32m 68\u001b[39m \u001b[38;5;66;03m# it and importing it first would fail if the OpenMP dll cannot be found.\u001b[39;00m\n\u001b[32m 69\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ( \u001b[38;5;66;03m# noqa: F401 E402\u001b[39;00m\n\u001b[32m 70\u001b[39m __check_build,\n\u001b[32m 71\u001b[39m _distributor_init,\n\u001b[32m 72\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbase\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m clone \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n\u001b[32m 74\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_show_versions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m show_versions \u001b[38;5;66;03m# noqa: E402\u001b[39;00m\n\u001b[32m 76\u001b[39m _submodules = [\n\u001b[32m 77\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcalibration\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 78\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcluster\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 114\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcompose\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 115\u001b[39m ]\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/base.py:19\u001b[39m\n\u001b[32m 17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_config\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexceptions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m InconsistentVersionWarning\n\u001b[32m---> \u001b[39m\u001b[32m19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_metadata_requests\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _MetadataRequester, _routing_enabled\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_missing\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m is_scalar_nan\n\u001b[32m 21\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_param_validation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m validate_parameter_constraints\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/__init__.py:9\u001b[39m\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m metadata_routing\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_bunch\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Bunch\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_chunking\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m gen_batches, gen_even_slices\n\u001b[32m 11\u001b[39m \u001b[38;5;66;03m# Make _safe_indexing importable from here for backward compat as this particular\u001b[39;00m\n\u001b[32m 12\u001b[39m \u001b[38;5;66;03m# helper is considered semi-private and typically very useful for third-party\u001b[39;00m\n\u001b[32m 13\u001b[39m \u001b[38;5;66;03m# libraries that want to comply with scikit-learn's estimator API. In particular,\u001b[39;00m\n\u001b[32m 14\u001b[39m \u001b[38;5;66;03m# _safe_indexing was included in our public API documentation despite the leading\u001b[39;00m\n\u001b[32m 15\u001b[39m \u001b[38;5;66;03m# `_` in its name.\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_indexing\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 17\u001b[39m _safe_indexing, \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[32m 18\u001b[39m resample,\n\u001b[32m 19\u001b[39m shuffle,\n\u001b[32m 20\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/_chunking.py:11\u001b[39m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m 10\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_config\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_config\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_param_validation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Interval, validate_params\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mchunk_generator\u001b[39m(gen, chunksize):\n\u001b[32m 15\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Chunk generator, ``gen`` into lists of length ``chunksize``. The last\u001b[39;00m\n\u001b[32m 16\u001b[39m \u001b[33;03m chunk may have a length less than ``chunksize``.\"\"\"\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/_param_validation.py:17\u001b[39m\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mscipy\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01msparse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m csr_matrix, issparse\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_config\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m config_context, get_config\n\u001b[32m---> \u001b[39m\u001b[32m17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mvalidation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _is_arraylike_not_scalar\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mInvalidParameterError\u001b[39;00m(\u001b[38;5;167;01mValueError\u001b[39;00m, \u001b[38;5;167;01mTypeError\u001b[39;00m):\n\u001b[32m 21\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Custom exception to be raised when the parameter of a class/method/function\u001b[39;00m\n\u001b[32m 22\u001b[39m \u001b[33;03m does not have a valid type or value.\u001b[39;00m\n\u001b[32m 23\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/validation.py:21\u001b[39m\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_config \u001b[38;5;28;01mas\u001b[39;00m _get_config\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexceptions\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DataConversionWarning, NotFittedError, PositiveSpectrumWarning\n\u001b[32m---> \u001b[39m\u001b[32m21\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01m_array_api\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _asarray_with_order, _is_numpy_namespace, get_namespace\n\u001b[32m 22\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdeprecation\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m _deprecate_force_all_finite\n\u001b[32m 23\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mfixes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ComplexWarning, _preserve_dia_indices_dtype\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/_array_api.py:20\u001b[39m\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexternals\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m array_api_extra \u001b[38;5;28;01mas\u001b[39;00m xpx\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexternals\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01marray_api_compat\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m numpy \u001b[38;5;28;01mas\u001b[39;00m np_compat\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mfixes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m parse_version\n\u001b[32m 22\u001b[39m \u001b[38;5;66;03m# TODO: complete __all__\u001b[39;00m\n\u001b[32m 23\u001b[39m __all__ = [\u001b[33m\"\u001b[39m\u001b[33mxpx\u001b[39m\u001b[33m\"\u001b[39m] \u001b[38;5;66;03m# we import xpx here just to re-export it, need this to appease ruff\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/sklearn/utils/fixes.py:20\u001b[39m\n\u001b[32m 17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mscipy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m optimize\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpd\u001b[39;00m\n\u001b[32m 21\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[32m 22\u001b[39m pd = \u001b[38;5;28;01mNone\u001b[39;00m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/__init__.py:49\u001b[39m\n\u001b[32m 46\u001b[39m \u001b[38;5;66;03m# let init-time option registration happen\u001b[39;00m\n\u001b[32m 47\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mconfig_init\u001b[39;00m \u001b[38;5;66;03m# pyright: ignore[reportUnusedImport] # noqa: F401\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m49\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 50\u001b[39m \u001b[38;5;66;03m# dtype\u001b[39;00m\n\u001b[32m 51\u001b[39m ArrowDtype,\n\u001b[32m 52\u001b[39m Int8Dtype,\n\u001b[32m 53\u001b[39m Int16Dtype,\n\u001b[32m 54\u001b[39m Int32Dtype,\n\u001b[32m 55\u001b[39m Int64Dtype,\n\u001b[32m 56\u001b[39m UInt8Dtype,\n\u001b[32m 57\u001b[39m UInt16Dtype,\n\u001b[32m 58\u001b[39m UInt32Dtype,\n\u001b[32m 59\u001b[39m UInt64Dtype,\n\u001b[32m 60\u001b[39m Float32Dtype,\n\u001b[32m 61\u001b[39m Float64Dtype,\n\u001b[32m 62\u001b[39m CategoricalDtype,\n\u001b[32m 63\u001b[39m PeriodDtype,\n\u001b[32m 64\u001b[39m IntervalDtype,\n\u001b[32m 65\u001b[39m DatetimeTZDtype,\n\u001b[32m 66\u001b[39m StringDtype,\n\u001b[32m 67\u001b[39m BooleanDtype,\n\u001b[32m 68\u001b[39m \u001b[38;5;66;03m# missing\u001b[39;00m\n\u001b[32m 69\u001b[39m NA,\n\u001b[32m 70\u001b[39m isna,\n\u001b[32m 71\u001b[39m isnull,\n\u001b[32m 72\u001b[39m notna,\n\u001b[32m 73\u001b[39m notnull,\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# indexes\u001b[39;00m\n\u001b[32m 75\u001b[39m Index,\n\u001b[32m 76\u001b[39m CategoricalIndex,\n\u001b[32m 77\u001b[39m RangeIndex,\n\u001b[32m 78\u001b[39m MultiIndex,\n\u001b[32m 79\u001b[39m IntervalIndex,\n\u001b[32m 80\u001b[39m TimedeltaIndex,\n\u001b[32m 81\u001b[39m DatetimeIndex,\n\u001b[32m 82\u001b[39m PeriodIndex,\n\u001b[32m 83\u001b[39m IndexSlice,\n\u001b[32m 84\u001b[39m \u001b[38;5;66;03m# tseries\u001b[39;00m\n\u001b[32m 85\u001b[39m NaT,\n\u001b[32m 86\u001b[39m Period,\n\u001b[32m 87\u001b[39m period_range,\n\u001b[32m 88\u001b[39m Timedelta,\n\u001b[32m 89\u001b[39m timedelta_range,\n\u001b[32m 90\u001b[39m Timestamp,\n\u001b[32m 91\u001b[39m date_range,\n\u001b[32m 92\u001b[39m bdate_range,\n\u001b[32m 93\u001b[39m Interval,\n\u001b[32m 94\u001b[39m interval_range,\n\u001b[32m 95\u001b[39m DateOffset,\n\u001b[32m 96\u001b[39m \u001b[38;5;66;03m# conversion\u001b[39;00m\n\u001b[32m 97\u001b[39m to_numeric,\n\u001b[32m 98\u001b[39m to_datetime,\n\u001b[32m 99\u001b[39m to_timedelta,\n\u001b[32m 100\u001b[39m \u001b[38;5;66;03m# misc\u001b[39;00m\n\u001b[32m 101\u001b[39m Flags,\n\u001b[32m 102\u001b[39m Grouper,\n\u001b[32m 103\u001b[39m factorize,\n\u001b[32m 104\u001b[39m unique,\n\u001b[32m 105\u001b[39m value_counts,\n\u001b[32m 106\u001b[39m NamedAgg,\n\u001b[32m 107\u001b[39m array,\n\u001b[32m 108\u001b[39m Categorical,\n\u001b[32m 109\u001b[39m set_eng_float_format,\n\u001b[32m 110\u001b[39m Series,\n\u001b[32m 111\u001b[39m DataFrame,\n\u001b[32m 112\u001b[39m )\n\u001b[32m 114\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdtypes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdtypes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m SparseDtype\n\u001b[32m 116\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtseries\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m infer_freq\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/api.py:47\u001b[39m\n\u001b[32m 45\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mconstruction\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m array\n\u001b[32m 46\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mflags\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Flags\n\u001b[32m---> \u001b[39m\u001b[32m47\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 48\u001b[39m Grouper,\n\u001b[32m 49\u001b[39m NamedAgg,\n\u001b[32m 50\u001b[39m )\n\u001b[32m 51\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mindexes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 52\u001b[39m CategoricalIndex,\n\u001b[32m 53\u001b[39m DatetimeIndex,\n\u001b[32m (...)\u001b[39m\u001b[32m 59\u001b[39m TimedeltaIndex,\n\u001b[32m 60\u001b[39m )\n\u001b[32m 61\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mindexes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdatetimes\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 62\u001b[39m bdate_range,\n\u001b[32m 63\u001b[39m date_range,\n\u001b[32m 64\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/groupby/__init__.py:1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgeneric\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[32m 2\u001b[39m DataFrameGroupBy,\n\u001b[32m 3\u001b[39m NamedAgg,\n\u001b[32m 4\u001b[39m SeriesGroupBy,\n\u001b[32m 5\u001b[39m )\n\u001b[32m 6\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m GroupBy\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgroupby\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mgrouper\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Grouper\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/groupby/generic.py:1329\u001b[39m\n\u001b[32m 1325\u001b[39m result = \u001b[38;5;28mself\u001b[39m._op_via_apply(\u001b[33m\"\u001b[39m\u001b[33munique\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 1326\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[32m-> \u001b[39m\u001b[32m1329\u001b[39m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[38;5;250;43m \u001b[39;49m\u001b[34;43;01mDataFrameGroupBy\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43mGroupBy\u001b[49m\u001b[43m[\u001b[49m\u001b[43mDataFrame\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 1330\u001b[39m \u001b[43m \u001b[49m\u001b[43m_agg_examples_doc\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mdedent\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1331\u001b[39m \u001b[38;5;250;43m \u001b[39;49m\u001b[33;43;03m\"\"\"\u001b[39;49;00m\n\u001b[32m 1332\u001b[39m \u001b[33;43;03m Examples\u001b[39;49;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 1417\u001b[39m \u001b[33;43;03m \"\"\"\u001b[39;49;00m\n\u001b[32m 1418\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1420\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;129;43m@doc\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m_agg_template_frame\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexamples\u001b[49m\u001b[43m=\u001b[49m\u001b[43m_agg_examples_doc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mklass\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mDataFrame\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 1421\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mdef\u001b[39;49;00m\u001b[38;5;250;43m \u001b[39;49m\u001b[34;43maggregate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.virtualenvs/coniferest/lib/python3.13/site-packages/pandas/core/groupby/generic.py:1758\u001b[39m, in \u001b[36mDataFrameGroupBy\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 1755\u001b[39m concatenated = concatenated.reindex(concat_index, axis=other_axis, copy=\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[32m 1756\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._set_result_index_ordered(concatenated)\n\u001b[32m-> \u001b[39m\u001b[32m1758\u001b[39m __examples_dataframe_doc = \u001b[43mdedent\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1759\u001b[39m \u001b[38;5;250;43m \u001b[39;49m\u001b[33;43;03m\"\"\"\u001b[39;49;00m\n\u001b[32m 1760\u001b[39m \u001b[33;43;03m>>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',\u001b[39;49;00m\n\u001b[32m 1761\u001b[39m \u001b[33;43;03m... 'foo', 'bar'],\u001b[39;49;00m\n\u001b[32m 1762\u001b[39m \u001b[33;43;03m... 'B' : ['one', 'one', 'two', 'three',\u001b[39;49;00m\n\u001b[32m 1763\u001b[39m \u001b[33;43;03m... 'two', 'two'],\u001b[39;49;00m\n\u001b[32m 1764\u001b[39m \u001b[33;43;03m... 'C' : [1, 5, 5, 2, 5, 5],\u001b[39;49;00m\n\u001b[32m 1765\u001b[39m \u001b[33;43;03m... 'D' : [2.0, 5., 8., 1., 2., 9.]})\u001b[39;49;00m\n\u001b[32m 1766\u001b[39m \u001b[33;43;03m>>> grouped = df.groupby('A')[['C', 'D']]\u001b[39;49;00m\n\u001b[32m 1767\u001b[39m \u001b[33;43;03m>>> grouped.transform(lambda x: (x - x.mean()) / x.std())\u001b[39;49;00m\n\u001b[32m 1768\u001b[39m \u001b[33;43;03m C D\u001b[39;49;00m\n\u001b[32m 1769\u001b[39m \u001b[33;43;03m0 -1.154701 -0.577350\u001b[39;49;00m\n\u001b[32m 1770\u001b[39m \u001b[33;43;03m1 0.577350 0.000000\u001b[39;49;00m\n\u001b[32m 1771\u001b[39m \u001b[33;43;03m2 0.577350 1.154701\u001b[39;49;00m\n\u001b[32m 1772\u001b[39m \u001b[33;43;03m3 -1.154701 -1.000000\u001b[39;49;00m\n\u001b[32m 1773\u001b[39m \u001b[33;43;03m4 0.577350 -0.577350\u001b[39;49;00m\n\u001b[32m 1774\u001b[39m \u001b[33;43;03m5 0.577350 1.000000\u001b[39;49;00m\n\u001b[32m 1775\u001b[39m \n\u001b[32m 1776\u001b[39m \u001b[33;43;03mBroadcast result of the transformation\u001b[39;49;00m\n\u001b[32m 1777\u001b[39m \n\u001b[32m 1778\u001b[39m \u001b[33;43;03m>>> grouped.transform(lambda x: x.max() - x.min())\u001b[39;49;00m\n\u001b[32m 1779\u001b[39m \u001b[33;43;03m C D\u001b[39;49;00m\n\u001b[32m 1780\u001b[39m \u001b[33;43;03m0 4.0 6.0\u001b[39;49;00m\n\u001b[32m 1781\u001b[39m \u001b[33;43;03m1 3.0 8.0\u001b[39;49;00m\n\u001b[32m 1782\u001b[39m \u001b[33;43;03m2 4.0 6.0\u001b[39;49;00m\n\u001b[32m 1783\u001b[39m \u001b[33;43;03m3 3.0 8.0\u001b[39;49;00m\n\u001b[32m 1784\u001b[39m \u001b[33;43;03m4 4.0 6.0\u001b[39;49;00m\n\u001b[32m 1785\u001b[39m \u001b[33;43;03m5 3.0 8.0\u001b[39;49;00m\n\u001b[32m 1786\u001b[39m \n\u001b[32m 1787\u001b[39m \u001b[33;43;03m>>> grouped.transform(\"mean\")\u001b[39;49;00m\n\u001b[32m 1788\u001b[39m \u001b[33;43;03m C D\u001b[39;49;00m\n\u001b[32m 1789\u001b[39m \u001b[33;43;03m0 3.666667 4.0\u001b[39;49;00m\n\u001b[32m 1790\u001b[39m \u001b[33;43;03m1 4.000000 5.0\u001b[39;49;00m\n\u001b[32m 1791\u001b[39m \u001b[33;43;03m2 3.666667 4.0\u001b[39;49;00m\n\u001b[32m 1792\u001b[39m \u001b[33;43;03m3 4.000000 5.0\u001b[39;49;00m\n\u001b[32m 1793\u001b[39m \u001b[33;43;03m4 3.666667 4.0\u001b[39;49;00m\n\u001b[32m 1794\u001b[39m \u001b[33;43;03m5 4.000000 5.0\u001b[39;49;00m\n\u001b[32m 1795\u001b[39m \n\u001b[32m 1796\u001b[39m \u001b[33;43;03m.. versionchanged:: 1.3.0\u001b[39;49;00m\n\u001b[32m 1797\u001b[39m \n\u001b[32m 1798\u001b[39m \u001b[33;43;03mThe resulting dtype will reflect the return value of the passed ``func``,\u001b[39;49;00m\n\u001b[32m 1799\u001b[39m \u001b[33;43;03mfor example:\u001b[39;49;00m\n\u001b[32m 1800\u001b[39m \n\u001b[32m 1801\u001b[39m \u001b[33;43;03m>>> grouped.transform(lambda x: x.astype(int).max())\u001b[39;49;00m\n\u001b[32m 1802\u001b[39m \u001b[33;43;03mC D\u001b[39;49;00m\n\u001b[32m 1803\u001b[39m \u001b[33;43;03m0 5 8\u001b[39;49;00m\n\u001b[32m 1804\u001b[39m \u001b[33;43;03m1 5 9\u001b[39;49;00m\n\u001b[32m 1805\u001b[39m \u001b[33;43;03m2 5 8\u001b[39;49;00m\n\u001b[32m 1806\u001b[39m \u001b[33;43;03m3 5 9\u001b[39;49;00m\n\u001b[32m 1807\u001b[39m \u001b[33;43;03m4 5 8\u001b[39;49;00m\n\u001b[32m 1808\u001b[39m \u001b[33;43;03m5 5 9\u001b[39;49;00m\n\u001b[32m 1809\u001b[39m \u001b[33;43;03m\"\"\"\u001b[39;49;00m\n\u001b[32m 1810\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1812\u001b[39m \u001b[38;5;129m@Substitution\u001b[39m(klass=\u001b[33m\"\u001b[39m\u001b[33mDataFrame\u001b[39m\u001b[33m\"\u001b[39m, example=__examples_dataframe_doc)\n\u001b[32m 1813\u001b[39m \u001b[38;5;129m@Appender\u001b[39m(_transform_template)\n\u001b[32m 1814\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mtransform\u001b[39m(\u001b[38;5;28mself\u001b[39m, func, *args, engine=\u001b[38;5;28;01mNone\u001b[39;00m, engine_kwargs=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 1815\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._transform(\n\u001b[32m 1816\u001b[39m func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs\n\u001b[32m 1817\u001b[39m )\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.13.1-linux-x86_64-gnu/lib/python3.13/textwrap.py:466\u001b[39m, in \u001b[36mdedent\u001b[39m\u001b[34m(text)\u001b[39m\n\u001b[32m 462\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m line \u001b[38;5;129;01mor\u001b[39;00m line.startswith(margin), \\\n\u001b[32m 463\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mline = \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m, margin = \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m % (line, margin)\n\u001b[32m 465\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m margin:\n\u001b[32m--> \u001b[39m\u001b[32m466\u001b[39m text = \u001b[43mre\u001b[49m\u001b[43m.\u001b[49m\u001b[43msub\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mr\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m(?m)^\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m+\u001b[49m\u001b[43m \u001b[49m\u001b[43mmargin\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 467\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m text\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.13.1-linux-x86_64-gnu/lib/python3.13/re/__init__.py:208\u001b[39m, in \u001b[36msub\u001b[39m\u001b[34m(pattern, repl, string, count, flags, *args)\u001b[39m\n\u001b[32m 202\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mwarnings\u001b[39;00m\n\u001b[32m 203\u001b[39m warnings.warn(\n\u001b[32m 204\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m'\u001b[39m\u001b[33mcount\u001b[39m\u001b[33m'\u001b[39m\u001b[33m is passed as positional argument\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 205\u001b[39m \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m, stacklevel=\u001b[32m2\u001b[39m\n\u001b[32m 206\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m208\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_compile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpattern\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43msub\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrepl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstring\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[31mKeyboardInterrupt\u001b[39m: " - ] - } - ], + "outputs": [], "source": [ "from collections import defaultdict\n", "\n", @@ -71,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "initial_id", "metadata": { "ExecuteTime": { @@ -156,7 +127,15 @@ "execution_count": null, "id": "929fd77b-3333-4937-90aa-d2804151d868", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/200 [00:00 Date: Thu, 10 Jul 2025 23:17:14 +0300 Subject: [PATCH 38/40] Add (temp) gz2.py --- docs/notebooks/gz2.py | 107 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/notebooks/gz2.py diff --git a/docs/notebooks/gz2.py b/docs/notebooks/gz2.py new file mode 100644 index 0000000..d74e5de --- /dev/null +++ b/docs/notebooks/gz2.py @@ -0,0 +1,107 @@ +from collections import defaultdict + +import matplotlib.pyplot as plt +import numpy as np +from tqdm import tqdm + +from coniferest.aadforest import AADForest +from coniferest.datasets import Dataset, DevNetDataset +from coniferest.isoforest import IsolationForest +from coniferest.label import Label +from coniferest.pineforest import PineForest +from coniferest.session.oracle import OracleSession, create_oracle_session + +class Compare: + models = { + 'Isolation Forest': IsolationForest, + 'AAD': AADForest, + 'Pine Forest': PineForest, + } + + def __init__(self, dataset: Dataset, *, iterations=100, n_jobs=-1, sampletrees_per_batch=1<<20): + self.model_kwargs = { + 'n_trees': 128, + 'sampletrees_per_batch': sampletrees_per_batch, + 'n_jobs': n_jobs, + } + self.session_kwargs = { + 'data': dataset.data, + 'labels': dataset.labels, + 'max_iterations': iterations, + } + self.results = {} + self.steps = np.arange(1, iterations + 1) + self.total_anomaly_fraction = np.mean(dataset.labels == Label.A) + + def get_sessions(self, random_seed): + model_kwargs = self.model_kwargs | {'random_seed': random_seed} + + return { + name: create_oracle_session(model=model(**model_kwargs), **self.session_kwargs) + for name, model in self.models.items() + } + + def run(self, random_seeds): + assert len(random_seeds) == len(set(random_seeds)), "random seeds must be different" + + results = defaultdict(dict) + + futures = [] + for random_seed in tqdm(random_seeds): + sessions = self.get_sessions(random_seed) + for name, session in sessions.items(): + session.run() + anomalies = np.cumsum(np.array(list(session.known_labels.values())) == Label.A) + results[name][random_seed] = anomalies + + self.results |= results + return self + + def plot(self, dataset_name: str, savefig=False): + plt.figure(figsize=(8, 6)) + plt.title(f'Dataset: {dataset_name}') + + for name, anomalies_dict in self.results.items(): + anomalies = np.stack(list(anomalies_dict.values())) + q5, median, q95 = np.quantile(anomalies, [0.05, 0.5, 0.95], axis=0) + + plt.plot(self.steps, median, alpha=0.75, label=name) + plt.fill_between(self.steps, q5, q95, alpha=0.5) + + plt.plot(self.steps, self.steps * self.total_anomaly_fraction, ls='--', color='grey', + label='Theoretical random') + + plt.xlabel('Iteration') + plt.ylabel('Number of anomalies') + plt.grid() + plt.legend() + if savefig: + plt.savefig(f'{dataset_name}.pdf') + + return self + +import pickle +from pathlib import Path + +import pandas as pd + +class GalaxyZoo2Dataset(Dataset): + def __init__(self, path: Path, *, anomaly_class='Class6.1', anomaly_threshold=0.9): + astronomaly = pd.read_parquet(path / "astronomaly.parquet") + self.data = astronomaly.drop(columns=['GalaxyID', 'anomaly']).to_numpy().copy(order='C') + ids = astronomaly['GalaxyID'].to_numpy() + + solutions = pd.read_csv(path / "training_solutions_rev1.csv", index_col="GalaxyID") + anomaly = solutions[anomaly_class][ids] >= anomaly_threshold + self.labels = np.full(anomaly.shape, Label.R) + self.labels[anomaly] = Label.A + + +seeds = range(12, 212) + +path = Path("/home/hombit/gz2") +dataset_obj = GalaxyZoo2Dataset(path) +compare_zoo = Compare(dataset_obj, iterations=100, n_jobs=1, sampletrees_per_batch=1<<16).run(seeds) +compare_zoo.plot("Galaxy Zoo 2 (Anything odd? 90%)", savefig=True) +with open("galaxyzoo2_compare.pickle", "wb") as fh: + pickle.dump(compare_zoo, fh) From cd85457a33f41e49f92ff7e79aeb49186c9aa15b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Jul 2025 20:17:42 +0000 Subject: [PATCH 39/40] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/coniferest/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coniferest/datasets/__init__.py b/src/coniferest/datasets/__init__.py index c33c7ce..4c8c1cd 100644 --- a/src/coniferest/datasets/__init__.py +++ b/src/coniferest/datasets/__init__.py @@ -159,7 +159,7 @@ def __init__(self, name: str): raise ValueError(f"Dataset {name} is not available. Available datasets are: {self.avialble_datasets}") self.name = name - + df = pd.read_csv(self._dataset_urls[name]) # Last column is for class, the rest are features From 6e740b5fe3642a5b9ca50e5bfb0197e3c68e0f3a Mon Sep 17 00:00:00 2001 From: Konstantin Malanchev Date: Fri, 11 Jul 2025 09:14:55 -0400 Subject: [PATCH 40/40] Make mutable borrow of result arrays safe --- rust/src/tree_traversal.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rust/src/tree_traversal.rs b/rust/src/tree_traversal.rs index 2180d24..f2c27d9 100644 --- a/rust/src/tree_traversal.rs +++ b/rust/src/tree_traversal.rs @@ -89,8 +89,8 @@ where Ok({ let paths = PyArray1::zeros(py, data_view.nrows(), false); - // SAFETY: this call invalidates other views, but it is the only view we need - let paths_view_mut = unsafe { paths.as_array_mut() }; + let mut paths_rw = paths.readwrite(); + let paths_view_mut = paths_rw.as_array_mut(); // Here we need to dispatch `data` and run the template function calc_paths_sum_impl( @@ -146,8 +146,8 @@ where false, ); - // SAFETY: this call invalidates other views, but it is the only view we need - let values_view = unsafe { values.as_array_mut() }; + let mut values_rw = values.readwrite(); + let values_view = values_rw.as_array_mut(); // Here we need to dispatch `data` and run the template function calc_paths_sum_transpose_impl( @@ -190,10 +190,10 @@ where let delta_sum = PyArray2::zeros(py, (data_view.nrows(), data_view.ncols()), false); let hit_count = PyArray2::zeros(py, (data_view.nrows(), data_view.ncols()), false); - // SAFETY: this call invalidates other views, but it is the only view we need - let delta_sum_view = unsafe { delta_sum.as_array_mut() }; - // SAFETY: this call invalidates other views, but it is the only view we need - let hit_count_view = unsafe { hit_count.as_array_mut() }; + let mut delta_sum_rw = delta_sum.readwrite(); + let delta_sum_view = delta_sum_rw.as_array_mut(); + let mut hit_count_rw = hit_count.readwrite(); + let hit_count_view = hit_count_rw.as_array_mut(); calc_feature_delta_sum_impl( selectors_view, @@ -234,8 +234,8 @@ where Ok({ let leafs = PyArray2::zeros(py, (data_view.nrows(), node_offsets_view.len() - 1), false); - // SAFETY: this call invalidates other views, but it is the only view we need - let leafs_view = unsafe { leafs.as_array_mut() }; + let mut leafs_rw = leafs.readwrite(); + let leafs_view = leafs_rw.as_array_mut(); calc_apply_impl( selectors_view,