OpenLineage Provider issue: scheduler shuts down after pickle of OpenLineageListener fails #40309

scheduler  | [2024-06-18T13:57:05.429-0400] {scheduler_job_runner.py:860} ERROR - Exception when executing SchedulerJob._run_scheduler_loop
scheduler  | Traceback (most recent call last):
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/airflow/jobs/scheduler_job_runner.py", line 843, in _execute
scheduler  | self._run_scheduler_loop()
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/airflow/jobs/scheduler_job_runner.py", line 975, in _run_scheduler_loop
scheduler  | num_queued_tis = self._do_scheduling(session)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/airflow/jobs/scheduler_job_runner.py", line 1051, in _do_scheduling
scheduler  | self._start_queued_dagruns(session)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/airflow/jobs/scheduler_job_runner.py", line 1391, in _start_queued_dagruns
scheduler  | dag_run.notify_dagrun_state_changed()
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/airflow/models/dagrun.py", line 980, in notify_dagrun_state_changed
scheduler  | get_listener_manager().hook.on_dag_run_running(dag_run=self, msg=msg)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/pluggy/_hooks.py", line 513, in __call__
scheduler  | return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/pluggy/_manager.py", line 120, in _hookexec
scheduler  | return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/pluggy/_callers.py", line 139, in _multicall
scheduler  | raise exception.with_traceback(exception.__traceback__)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/pluggy/_callers.py", line 103, in _multicall
scheduler  | res = hook_impl.function(*args)
scheduler  | File "/Users/michael/Library/Python/3.9/lib/python/site-packages/airflow/providers/openlineage/plugins/listener.py", line 323, in on_dag_run_running
scheduler  | self.executor.submit(
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/process.py", line 697, in submit
scheduler  | self._adjust_process_count()
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/process.py", line 675, in _adjust_process_count
scheduler  | p.start()
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/multiprocessing/process.py", line 121, in start
scheduler  | self._popen = self._Popen(self)
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/multiprocessing/context.py", line 284, in _Popen
scheduler  | return Popen(process_obj)
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 32, in __init__
scheduler  | super().__init__(process_obj)
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/multiprocessing/popen_fork.py", line 19, in __init__
scheduler  | self._launch(process_obj)
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 47, in _launch
scheduler  | reduction.dump(process_obj, fp)
scheduler  | File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/multiprocessing/reduction.py", line 60, in dump
scheduler  | ForkingPickler(file, protocol).dump(obj)
scheduler  | AttributeError: Can't pickle local object 'OpenLineageListener.executor.<locals>.initializer'