From bf150e442eaed9af6e6f559cb51ba0cd96dded81 Mon Sep 17 00:00:00 2001
From: Barabazs <31799121+Barabazs@users.noreply.github.com>
Date: Thu, 2 Oct 2025 16:06:38 +0000
Subject: [PATCH] feat: update Punkt tokenizer to use pre-trained model and
 handle missing data

---
 whisperx/alignment.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/whisperx/alignment.py b/whisperx/alignment.py
index 3e19292..ae9f997 100644
--- a/whisperx/alignment.py
+++ b/whisperx/alignment.py
@@ -22,9 +22,8 @@ from whisperx.types import (
     SingleWordSegment,
     SegmentData,
 )
-from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters
-
-PUNKT_ABBREVIATIONS = ['dr', 'vs', 'mr', 'mrs', 'prof', 'jr', 'sr', 'ph.d']
+import nltk
+from nltk.data import load as nltk_load
 
 LANGUAGES_WITHOUT_SPACES = ["ja", "zh"]
 
@@ -188,9 +187,11 @@ def align(
                 clean_wdx.append(wdx)
 
 
-        punkt_param = PunktParameters()
-        punkt_param.abbrev_types = set(PUNKT_ABBREVIATIONS)
-        sentence_splitter = PunktSentenceTokenizer(punkt_param)
+        try:
+            sentence_splitter = nltk_load('tokenizers/punkt/english.pickle')
+        except LookupError:
+            nltk.download('punkt_tab', quiet=True)
+            sentence_splitter = nltk_load('tokenizers/punkt/english.pickle')
         sentence_spans = list(sentence_splitter.span_tokenize(text))
 
         segment_data[sdx] = {