missed files on previous commit

Author: misko

spf/dataset/segmentation.py

@@ -35,7 +35,7 @@
     "window_size": 2048,
     "stride": 2048,
     "trim": 20.0,
-    "mean_diff_threshold": 0.2,
+    # "mean_diff_threshold": 0.2,
     "max_stddev_threshold": 0.5,
     "drop_less_than_size": 3000,
     "min_abs_signal": 40,
@@ -539,7 +539,17 @@ def segment_session(
         # 4. Identifies windows as "signal" vs "noise" based on phase stability and amplitude
         # 5. Combines adjacent windows with similar phase characteristics
         # 6. Returns a list of segment information and signal statistics
-        segmentation_results.update(simple_segment(v, **kwrgs))
+        segmentation_results.update(
+            simple_segment(
+                v=v,
+                window_size=kwrgs["window_size"],
+                stride=kwrgs["stride"],
+                trim=kwrgs["trim"],
+                max_stddev_threshold=kwrgs["max_stddev_threshold"],
+                drop_less_than_size=kwrgs["drop_less_than_size"],
+                min_abs_signal=kwrgs["min_abs_signal"],
+            )
+        )
 
         # Transpose the window statistics for easier processing
         # all_windows_stats shape is (3, N_windows) where:
@@ -592,7 +602,12 @@ def segment_session(
             # If no signal windows were identified, use placeholder values
             segmentation_results["weighted_windows_stats"] = np.array([-1, -1, -1])
     else:
-        segmentation_results['all_windows_stats']=get_all_windows_stats(v=v,window_size=kwrgs['window_size'],stride=kwrgs['stride'],trim=kwrgs['trim'])[1]
+        segmentation_results["all_windows_stats"] = get_all_windows_stats(
+            v=v,
+            window_size=kwrgs["window_size"],
+            stride=kwrgs["stride"],
+            trim=kwrgs["trim"],
+        )[1]
 
         # Transpose the window statistics for easier processing
         # all_windows_stats shape is (3, N_windows) where:
@@ -611,6 +626,7 @@ def segment_session(
 
     return segmentation_results
 
+
 def get_all_windows_stats(
     v,
     window_size,
@@ -633,6 +649,7 @@ def get_all_windows_stats(
     )
     return step_idxs, step_stats
 
+
 def simple_segment(
     v,
     window_size,
@@ -691,7 +708,9 @@ def simple_segment(
     # window_idxs_and_stats = windowed_trimmed_circular_mean_and_stddev(
     #     v, pd, window_size=window_size, stride=stride, trim=trim
     # )
-    window_idxs_and_stats = get_all_windows_stats(v=v,window_size=window_size,stride=stride,trim=trim)
+    window_idxs_and_stats = get_all_windows_stats(
+        v=v, window_size=window_size, stride=stride, trim=trim
+    )
     # window_idxs_and_stats has two components:
     # [0] = list of window indices (start_idx, end_idx)
     # [1] = array of statistics (trimmed_mean, trimmed_stddev, abs_signal_median)

spf/dataset/spf_dataset.py

@@ -1,14 +1,14 @@
-import threading
 import logging
 import multiprocessing
 import os
 import pickle
+import queue
+import threading
 import time
 from contextlib import contextmanager
 from enum import Enum
 from functools import cache
 from typing import Dict, List
-import queue
 
 import numpy as np
 import torch
@@ -118,6 +118,9 @@ def v5_collate_keys_fast(keys: List[str], batch: List[List[Dict[str, torch.Tenso
         )
         if key == "windowed_beamformer" or key == "all_windows_stats":
             d[key] = d[key].to(torch.float32)
+    # d["dropout_mask_rand_values"] = torch.rand(
+    #     (*d["y_rad"].shape,8), device=d["y_rad"].device
+    # )
     # d["random_rotations"] = torch_pi_norm(
     #     torch.rand(d["y_rad"].shape[0], 1) * 2 * torch.pi
     # )
@@ -478,7 +481,7 @@ def __init__(
         self.target_ntheta = self.nthetas if target_ntheta is None else target_ntheta
 
         self.realtime = realtime
-        self.max_store_size=max_store_size
+        self.max_store_size = max_store_size
 
         self.max_in_memory = max_in_memory
         self.min_idx = 0
@@ -597,7 +600,7 @@ def __init__(
             self.empirical_data = None
         self.serving_idx = -1
 
-        #setup the reader thread
+        # setup the reader thread
         self.stop_event = threading.Event()
         self.reader_thread = threading.Thread(target=self._reader_loop, daemon=True)
         self.reader_thread.start()
@@ -614,16 +617,25 @@ def _reader_loop(self):
                         }
                     self.store[idx]["data"][ridx] = rendered_data
                     self.store[idx]["count"] += 1
-                    print("LENGTH OF STORE IS",len(self.store))
-                    if self.max_store_size is not None and len(self.store)>self.max_store_size:
-                        for idx in sorted(self.store.keys())[:-self.max_store_size]:
+                    print("LENGTH OF STORE IS", len(self.store))
+                    if (
+                        self.max_store_size is not None
+                        and len(self.store) > self.max_store_size
+                    ):
+                        for idx in sorted(self.store.keys())[: -self.max_store_size]:
                             self.store.pop(idx)
-                            print("POPPING",idx)
+                            print("POPPING", idx)
 
             except queue.Empty:
                 pass  # No new item, just keep looping
+            except (OSError, EOFError):
+                # Queue closed because process exiting. Exit silently.
+                break
             except Exception as e:
-                logging.exception("Error in v5inferencedataset queue reader thread")
+                if not self.stop_event.is_set():
+                    logging.exception("Error in v5inferencedataset queue reader thread")
+                # If stop_event is set, don't log, just exit.
+                break
 
     def __len__(self):
         return self.serving_idx
@@ -724,7 +736,7 @@ def render_session(self, ridx, data):
         #     ]
         # ).T # torch.Size([1, 2])
 
-        data["rx_pos_mm"] = data["tx_pos_mm"] = torch.ones(1, 2) * torch.nan
+        data["rx_pos_mm"] = data["tx_pos_mm"] = torch.zeros(1, 2)  # * torch.nan
 
         data["rx_pos_xy"] = (
             data["rx_pos_mm"][snapshot_idxs].unsqueeze(0) / self.distance_normalization
@@ -789,6 +801,15 @@ def render_session(self, ridx, data):
                 data[key] = data[key].to(self.target_dtype)
         return data
 
+    def close(self):
+        if hasattr(self, "stop_event"):
+            self.stop_event.set()
+        if hasattr(self, "reader_thread") and self.reader_thread.is_alive():
+            self.reader_thread.join(timeout=1.0)  # avoid blocking forever
+
+    def __del__(self):
+        self.close()
+
 
 class v5spfdataset(Dataset):
     def __init__(

spf/model_training_and_inference/models/single_point_networks.py

@@ -12,12 +12,12 @@
 TEMP = 10
 
 
-@torch.jit.script
+# @torch.jit.script
 def cdf(mean: torch.Tensor, sigma: torch.Tensor, value: float):
     return 0.5 * (1 + torch.erf((value - mean) * sigma.reciprocal() / sqrt(2)))
 
 
-@torch.jit.script
+# @torch.jit.script
 def normal_correction_for_bounded_range(
     mean: torch.Tensor, sigma: torch.Tensor, max_y: float
 ):
@@ -468,10 +468,16 @@ def __init__(self, model_config, global_config):
         )
 
     def prepare_input(self, batch, additional_inputs=[]):
-        dropout_mask = (
-            torch.rand((8, *batch["y_rad"].shape), device=batch["y_rad"].device)
-            < self.input_dropout
-        )
+        if self.training:
+            # dropout_mask = (
+            #     torch.rand((8, *batch["y_rad"].shape), device=batch["y_rad"].device)
+            #     < self.input_dropout
+            # )
+            shape = batch["y_rad"].shape
+            rand_shape = torch.Size((8, shape[0], shape[1]))
+            dropout_mask = torch.rand(rand_shape, device=batch["y_rad"].device).to(
+                torch.bool
+            )
         # 1 , 65
         # if mask out 65, then scale up 1 by 65?
         # [ batch, samples, dim of input ]
@@ -976,6 +982,7 @@ def forward(self, batch):
             additional_inputs.append(
                 self.signal_matrix_net(input_with_spacing.select(1, 0))[:, None]
             )
+        # print(batch["all_windows_stats"].abs().mean())
         return_dict["single"] = torch.nn.functional.normalize(
             self.single_point_with_beamformer_ffnn(
                 self.prepare_input.prepare_input(batch, additional_inputs),

spf/model_training_and_inference/models/single_point_networks_inference.py

@@ -54,6 +54,7 @@ def single_example_realtime_inference(model, global_config, optim_config, realti
 
 
 def single_example_inference(model, global_config, datasets_config, optim_config):
+    model.eval()
 
     ds = v5spfdataset(
         datasets_config["train_paths"][0],

spf/rf.py

@@ -55,7 +55,7 @@ def pi_norm_halfpi(x):
     return ((x + np.pi / 2) % (2 * np.pi / 2)) - np.pi / 2
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_circular_diff_to_mean(angles: torch.Tensor, means: torch.Tensor):
     assert means.ndim == 1
     a = torch.abs(means[:, None] - angles) % (2 * torch.pi)
@@ -68,12 +68,12 @@ def torch_circular_diff_to_mean(angles: torch.Tensor, means: torch.Tensor):
 #     return ((x + max_angle) % (2 * max_angle)) - max_angle
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_pi_norm_pi(x):
     return ((x + torch.pi) % (2 * torch.pi)) - torch.pi
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_pi_norm(x: torch.Tensor, max_angle: float = torch.pi):
     return ((x + max_angle) % (2 * max_angle)) - max_angle
 
@@ -102,7 +102,7 @@ def circular_stddev(v, u, trim=50.0):
 
 
 # returns circular_stddev and trimmed cricular stddev
-@torch.jit.script
+# @torch.jit.script
 def torch_circular_stddev(v: torch.Tensor, u: torch.Tensor, trim: float):  # =50.0):
     diff_from_mean = torch_circular_diff_to_mean(angles=v, means=u.reshape(-1))
 
@@ -126,7 +126,7 @@ def torch_circular_stddev(v: torch.Tensor, u: torch.Tensor, trim: float):  # =50
     return stddev, trimmed_stddev
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_reduce_theta_to_positive_y(ground_truth_thetas):
     reduced_thetas = ground_truth_thetas.clone()
     # |theta|>np.pi/2 means its on the y<0
@@ -236,7 +236,7 @@ def circular_mean_single(angles, trim, weights=None):
     return pi_norm(cm), pi_norm(_cm)
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_circular_mean_notrim(angles: torch.Tensor):
     assert angles.ndim == 2
     _sin_angles = torch.sin(angles)
@@ -247,7 +247,7 @@ def torch_circular_mean_notrim(angles: torch.Tensor):
     return r, r
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_circular_mean_noweight(angles: torch.Tensor, trim: float):
     assert angles.ndim == 2
     _sin_angles = torch.sin(angles)
@@ -315,7 +315,7 @@ def torch_circular_mean(angles: torch.Tensor, trim: float, weights=None):
     return torch_pi_norm_pi(cm), torch_pi_norm_pi(_cm)
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_get_stats_for_signal(v: torch.Tensor, pd: torch.Tensor, trim: float):
     trimmed_cm = torch_circular_mean_noweight(pd.reshape(1, -1), trim=trim)[1][
         0
@@ -335,7 +335,7 @@ def get_stats_for_signal(v, pd, trim):
     return trimmed_cm, trimmed_stddev, abs_signal_median
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_windowed_trimmed_circular_mean_and_stddev(
     v: torch.Tensor, pd: torch.Tensor, window_size: int, stride: int, trim: float
 ):
@@ -415,7 +415,7 @@ def get_phase_diff(signal_matrix):
     return pi_norm(np.angle(signal_matrix[0]) - np.angle(signal_matrix[1]))
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_get_phase_diff(signal_matrix: torch.Tensor):
     return torch_pi_norm_pi(signal_matrix[:, 0].angle() - signal_matrix[:, 1].angle())
 
@@ -427,7 +427,7 @@ def get_avg_phase(signal_matrix, trim=0.0):
     ).reshape(-1)
 
 
-@torch.jit.script
+# @torch.jit.script
 def torch_get_avg_phase_notrim(signal_matrix: torch.Tensor):
     return torch.hstack(
         torch_circular_mean_notrim(