diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..6e803bf24 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -153,6 +153,9 @@ def __init__( elif lang == 'hu': from nemo_text_processing.text_normalization.hu.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.hu.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'pl': + from nemo_text_processing.text_normalization.pl.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.pl.verbalizers.verbalize_final import VerbalizeFinalFst elif lang == 'zh': from nemo_text_processing.text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.zh.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/pl/__init__.py b/nemo_text_processing/text_normalization/pl/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/__init__.py b/nemo_text_processing/text_normalization/pl/data/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/abbreviations.tsv b/nemo_text_processing/text_normalization/pl/data/abbreviations.tsv new file mode 100644 index 000000000..df8e38c75 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/abbreviations.tsv @@ -0,0 +1 @@ +s-ka spółka noun_f_ka.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv b/nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv new file mode 100644 index 000000000..63c3c6f17 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv @@ -0,0 +1,3 @@ +tzw. tak zwany +ur. urodzony +ang. angielski diff --git a/nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv b/nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv new file mode 100644 index 000000000..1c09cac2d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv @@ -0,0 +1 @@ +r. rok noun_mi_rok.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/dates/__init__.py b/nemo_text_processing/text_normalization/pl/data/dates/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv b/nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv new file mode 100644 index 000000000..66b0c5406 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv @@ -0,0 +1,12 @@ +sty stycznia +lut lutego +mar marca +kwi kwietnia +maj maja +cze czerwca +lip lipca +sie sierpnia +wrz września +paź października +lis listopada +gru grudnia diff --git a/nemo_text_processing/text_normalization/pl/data/dates/months.tsv b/nemo_text_processing/text_normalization/pl/data/dates/months.tsv new file mode 100644 index 000000000..d8c99aee5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 stycznia +2 lutego +3 marca +4 kwietnia +5 maja +6 czerwca +7 lipca +8 sierpnia +9 września +10 października +11 listopada +12 grudnia diff --git a/nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv b/nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv new file mode 100644 index 000000000..152aa9dce --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv @@ -0,0 +1,12 @@ +I stycznia +II lutego +III marca +IV kwietnia +V maja +VI czerwca +VII lipca +VIII sierpnia +IX września +X października +XI listopada +XII grudnia diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py b/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ga.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ga.tsv new file mode 100644 index 000000000..a38d46bed --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ga.tsv @@ -0,0 +1,14 @@ +sg_nom ga +sg_gen gi +sg_dat dze +sg_acc gę +sg_ins gą +sg_loc dze +sg_voc go +pl_nom gi +pl_gen g +pl_dat gom +pl_acc gi +pl_ins gami +pl_loc gach +pl_voc gi diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ia.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ia.tsv new file mode 100644 index 000000000..e07da049e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ia.tsv @@ -0,0 +1,14 @@ +sg_nom ia +sg_gen ii +sg_dat ii +sg_acc ię +sg_ins ią +sg_loc ii +sg_voc io +pl_nom ie +pl_gen ii +pl_dat iom +pl_acc ie +pl_ins iami +pl_loc iach +pl_voc ie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ka.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ka.tsv new file mode 100644 index 000000000..2f53858c2 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ka.tsv @@ -0,0 +1,14 @@ +sg_nom ka +sg_gen ki +sg_dat ce +sg_acc kę +sg_ins ką +sg_loc ce +sg_voc ko +pl_nom ki +pl_gen ek +pl_dat kom +pl_acc ki +pl_ins kami +pl_loc kach +pl_voc ki diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_la.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_la.tsv new file mode 100644 index 000000000..0d04494cd --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_la.tsv @@ -0,0 +1,14 @@ +sg_nom la +sg_gen li +sg_dat li +sg_acc lę +sg_ins lą +sg_loc li +sg_voc lo +pl_nom le +pl_gen l +pl_dat lom +pl_acc li +pl_ins lami +pl_loc lach +pl_voc le diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_na.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_na.tsv new file mode 100644 index 000000000..c32ae067b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_na.tsv @@ -0,0 +1,14 @@ +sg_nom na +sg_gen ny +sg_dat nie +sg_acc nę +sg_ins ną +sg_loc nie +sg_voc no +pl_nom ny +pl_gen n +pl_dat nom +pl_acc ny +pl_ins nami +pl_loc nach +pl_voc ny \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ta.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ta.tsv new file mode 100644 index 000000000..33823b746 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ta.tsv @@ -0,0 +1,14 @@ +sg_nom ta +sg_gen ty +sg_dat cie +sg_acc tę +sg_ins tą +sg_loc cie +sg_voc to +pl_nom ty +pl_gen t +pl_dat tom +pl_acc ty +pl_ins tami +pl_loc tach +pl_voc ty diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv new file mode 100644 index 000000000..e2ef6ef01 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc +sg_ins em +sg_loc u +sg_voc u +pl_nom e +pl_gen ów +pl_dat om +pl_acc e +pl_ins ami +pl_loc ach +pl_voc e diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv new file mode 100644 index 000000000..fd45b8826 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc +sg_ins iem +sg_loc u +sg_voc u +pl_nom i +pl_gen ów +pl_dat om +pl_acc i +pl_ins ami +pl_loc ach +pl_voc i diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv new file mode 100644 index 000000000..990a46dd0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen u +sg_dat owi +sg_acc +sg_ins iem +sg_loc u +sg_voc u +pl_nom i +pl_gen ów +pl_dat om +pl_acc i +pl_ins ami +pl_loc ach +pl_voc i diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv new file mode 100644 index 000000000..fba1a3200 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc +sg_ins em +sg_loc ze +sg_voc ze +pl_nom y +pl_gen ów +pl_dat om +pl_acc y +pl_ins ami +pl_loc ach +pl_voc y diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv new file mode 100644 index 000000000..fcaa88061 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc zie +sg_voc zie +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_ek.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_ek.tsv new file mode 100644 index 000000000..369e379ac --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_ek.tsv @@ -0,0 +1,14 @@ +sg_nom ek +sg_gen ka +sg_dat kowi +sg_acc ka +sg_ins kiem +sg_loc ku +sg_voc ku +pl_nom kowie +pl_gen ków +pl_dat kom +pl_acc ków +pl_ins kami +pl_loc kach +pl_voc kowie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv new file mode 100644 index 000000000..a9b2ca611 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv @@ -0,0 +1,14 @@ +sg_nom eł +sg_gen ła +sg_dat łowi +sg_acc ła +sg_ins łem +sg_loc le +sg_voc le +pl_nom łowie +pl_gen łów +pl_dat łom +pl_acc łów +pl_ins łami +pl_loc łach +pl_voc łowie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_i.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_i.tsv new file mode 100644 index 000000000..66bae2274 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_i.tsv @@ -0,0 +1,14 @@ +sg_nom i +sg_gen ego +sg_dat emu +sg_acc ego +sg_ins im +sg_loc im +sg_voc i +pl_nom owie +pl_gen ich +pl_dat im +pl_acc ich +pl_ins imi +pl_loc ich +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv new file mode 100644 index 000000000..c35470aa7 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins iem +sg_loc u +sg_voc u +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv new file mode 100644 index 000000000..0f86fa546 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc ie +sg_voc ie +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_rz.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_rz.tsv new file mode 100644 index 000000000..93302893e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_rz.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc u +sg_voc u +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_y.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_y.tsv new file mode 100644 index 000000000..88dc0860d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_y.tsv @@ -0,0 +1,14 @@ +sg_nom y +sg_gen ego +sg_dat emu +sg_acc ego +sg_ins ym +sg_loc ym +sg_voc y +pl_nom owie +pl_gen ych +pl_dat ym +pl_acc ych +pl_ins ymi +pl_loc ych +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv new file mode 100644 index 000000000..24a67362e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv @@ -0,0 +1,7 @@ +pl_nom e +pl_gen ga +pl_dat gu +pl_acc e +pl_ins giem +pl_loc gu +pl_voc e diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv new file mode 100644 index 000000000..aac94d8b8 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv @@ -0,0 +1,14 @@ +sg_nom o +sg_gen a +sg_dat u +sg_acc o +sg_ins em +sg_loc ze +sg_voc o +pl_nom a +pl_gen +pl_dat om +pl_acc a +pl_ins ami +pl_loc ach +pl_voc a diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv new file mode 100644 index 000000000..59865cf42 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv @@ -0,0 +1,21 @@ +bez gen +dla gen +do gen +od gen +u gen +z gen,ins +dzięki dat +ku dat +przeciw dat +wbrew dat +na acc,loc +o acc,loc +po acc,loc +przez acc +nad acc,ins +pod acc,ins +przed acc,ins +za acc,ins +między acc,ins +przy loc +w loc diff --git a/nemo_text_processing/text_normalization/pl/data/measures/__init__.py b/nemo_text_processing/text_normalization/pl/data/measures/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/measures/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/measures/units.tsv b/nemo_text_processing/text_normalization/pl/data/measures/units.tsv new file mode 100644 index 000000000..843f39f6f --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/measures/units.tsv @@ -0,0 +1,5 @@ +Hz herc noun_mi_c.tsv +kHz kiloherc noun_mi_c.tsv +m metr noun_mi_tr.tsv +cm centymetr noun_mi_tr.tsv +mm milimetr noun_mi_tr.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv new file mode 100644 index 000000000..840057285 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv @@ -0,0 +1,7 @@ +trzy 3 +cztery 4 +pięć 5 +sześć 6 +siedem 7 +osiem 8 +dziewięć 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv new file mode 100644 index 000000000..6bcc308e7 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv @@ -0,0 +1,56 @@ +2 mi_pl_nom dwa +2 mp_pl_nom dwaj +2 f_pl_nom dwie +2 pl_gen dwóch +2 pl_gen dwu +2 f_pl_ins dwiema +2 f_pl_ins dwoma +2 f_pl_ins dwu +2 mi_pl_ins dwoma +2 mi_pl_ins dwu +2 pl_dat dwóm +2 pl_dat dwom +2 pl_dat dwu +2 compound dwu +3 mi_pl_nom trzy +3 mp_pl_nom trzej +3 pl_gen trzech +3 pl_ins trzema +3 pl_dat trzem +3 compound trzy +4 mi_pl_nom cztery +4 mp_pl_nom czterej +4 pl_gen czterech +4 pl_ins czterema +4 pl_dat czterem +4 compound cztero +5 mi_pl_nom pięć +5 mp_pl_nom pięciu +5 pl_gen pięciu +5 pl_ins pięcioma +5 pl_dat pięciu +5 compound pięcio +6 mi_pl_nom sześć +6 mp_pl_nom sześciu +6 pl_gen sześciu +6 pl_ins sześcioma +6 pl_dat sześciu +6 compound sześcio +7 mi_pl_nom siedem +7 mp_pl_nom siedmiu +7 pl_gen siedmiu +7 pl_ins siedmioma +7 pl_dat siedmiu +7 compound siedmio +8 mi_pl_nom osiem +8 mp_pl_nom ośmiu +8 pl_gen ośmiu +8 pl_ins ośmioma +8 pl_dat ośmiu +8 compound ośmio +9 mi_pl_nom dziewięć +9 mp_pl_nom dziewięciu +9 pl_gen dziewięciu +9 pl_ins dziewięcioma +9 pl_dat dziewięciu +9 compound dziewięcio diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv new file mode 100644 index 000000000..dafa51f38 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv @@ -0,0 +1,9 @@ +jedynka 1 +dwójka 2 +trójka 3 +czwórka 4 +piątka 5 +szóstka 6 +siódemka 7 +ósemka 8 +dziewiątka 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv new file mode 100644 index 000000000..ae80ccc50 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv @@ -0,0 +1,9 @@ +jedno 1 +dwu 2 +trzy 3 +cztero 4 +pięcio 5 +sześcio 6 +siedmio 7 +ośmio 8 +dziewięcio 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv new file mode 100644 index 000000000..f1a25cd56 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv @@ -0,0 +1,4 @@ +dwój 2 +trój 3 +czworo 4 +czwór 4 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv new file mode 100644 index 000000000..d5532f424 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv @@ -0,0 +1,13 @@ +mi_pl_nom dwa +mp_pl_nom dwaj +f_pl_nom dwie +pl_gen dwóch +pl_gen dwu +f_pl_ins dwiema +f_pl_ins dwoma +f_pl_ins dwu +mi_pl_ins dwoma +mi_pl_ins dwu +pl_dat dwóm +pl_dat dwom +pl_dat dwu diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv new file mode 100644 index 000000000..6c29e26a0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv @@ -0,0 +1,9 @@ +sto 1 +dwieście 2 +trzysta 3 +czterysta 4 +pięćset 5 +sześćset 6 +siedemset 7 +osiemset 8 +dziewięćset 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv new file mode 100644 index 000000000..84b1ab48e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv @@ -0,0 +1,9 @@ +stu 1 +dwustu 2 +trzystu 3 +czterystu 4 +pięciuset 5 +sześciuset 6 +siedmiuset 7 +ośmiuset 8 +dziewięciuset 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv new file mode 100644 index 000000000..fb9209677 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv @@ -0,0 +1,9 @@ +stoma 1 +dwustoma 2 +trzystoma 3 +czterystoma 4 +pięciuset 5 +sześciuset 6 +siedmiuset 7 +ośmiuset 8 +dziewięciuset 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv new file mode 100644 index 000000000..f4e292b84 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv @@ -0,0 +1,9 @@ +setka 1 +dwusetka 2 +trzysetka 3 +czterysetka 4 +pięćsetka 5 +sześćsetka 6 +siedemsetka 7 +osiemsetka 8 +dziewięćsetka 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv new file mode 100644 index 000000000..1d812db0b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +dziesięć 10 +jedenaście 11 +dwanaście 12 +trzynaście 13 +czternaście 14 +piętnaście 15 +szesnaście 16 +siedemnaście 17 +osiemnaście 18 +dziewiętnaście 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv new file mode 100644 index 000000000..7845f6d55 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv @@ -0,0 +1,10 @@ +dziesięciu 10 +jedenastu 11 +dwunastu 12 +trzynastu 13 +czternastu 14 +piętnastu 15 +szesnastu 16 +siedemnastu 17 +osiemnastu 18 +dziewiętnastu 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv new file mode 100644 index 000000000..0a2d28adb --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv @@ -0,0 +1,10 @@ +dziesięcioma 10 +jedenastoma 11 +dwunastoma 12 +trzynastoma 13 +czternastoma 14 +piętnastoma 15 +szesnastoma 16 +siedemnastoma 17 +osiemnastoma 18 +dziewiętnastoma 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv new file mode 100644 index 000000000..f9617fe63 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv @@ -0,0 +1,10 @@ +dziesiątka 10 +jedenastka 11 +dwunastka 12 +trzynastka 13 +czternastka 14 +piętnastka 15 +szesnastka 16 +siedemnastka 17 +osiemnastka 18 +dziewiętnastka 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv new file mode 100644 index 000000000..52972da1e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv @@ -0,0 +1,10 @@ +dziesięcio 10 +jedenasto 11 +dwunasto 12 +trzynasto 13 +czternasto 14 +piętnasto 15 +szesnasto 16 +siedemnasto 17 +osiemnasto 18 +dziewiętnasto 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv new file mode 100644 index 000000000..aedd4ca9d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv @@ -0,0 +1,60 @@ +10 mi_pl_nom dziesięć +10 mp_pl_nom dziesięciu +10 pl_gen dziesięciu +10 pl_ins dziesięcioma +10 pl_dat dziesięciu +10 compound dziesięcio +11 mi_pl_nom jedenaście +11 mp_pl_nom jedenastu +11 pl_gen jedenastu +11 pl_ins jedenastoma +11 pl_dat jedenastu +11 compound jedenasto +12 mi_pl_nom dwanaście +12 mp_pl_nom dwunastu +12 pl_gen dwunastu +12 pl_ins dwunastoma +12 pl_dat dwunastu +12 compound dwunasto +13 mi_pl_nom trzynaście +13 mp_pl_nom trzynastu +13 pl_gen trzynastu +13 pl_ins trzynastoma +13 pl_dat trzynastu +13 compound trzynasto +14 mi_pl_nom czternaście +14 mp_pl_nom czternastu +14 pl_gen czternastu +14 pl_ins czternastoma +14 pl_dat czternastu +14 compound czternasto +15 mi_pl_nom piętnaście +15 mp_pl_nom piętnastu +15 pl_gen piętnastu +15 pl_ins piętnastoma +15 pl_dat piętnastu +15 compound piętnasto +16 mi_pl_nom szesnaście +16 mp_pl_nom szesnastu +16 pl_gen szesnastu +16 pl_ins szesnastoma +16 pl_dat szesnastu +16 compound szesnasto +17 mi_pl_nom siedemnaście +17 mp_pl_nom siedemnastu +17 pl_gen siedemnastu +17 pl_ins siedemnastoma +17 pl_dat siedemnastu +17 compound siedemnasto +18 mi_pl_nom osiemnaście +18 mp_pl_nom osiemnastu +18 pl_gen osiemnastu +18 pl_ins osiemnastoma +18 pl_dat osiemnastu +18 compound osiemnasto +19 mi_pl_nom dziewiętnaście +19 mp_pl_nom dziewiętnastu +19 pl_gen dziewiętnastu +19 pl_ins dziewiętnastoma +19 pl_dat dziewiętnastu +19 compound dziewiętnasto diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv new file mode 100644 index 000000000..2bb0a3a43 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +dwadzieścia 2 +trzydzieści 3 +czterdzieści 4 +pięćdziesiąt 5 +sześćdziesiąt 6 +siedemdziesiąt 7 +osiemdziesiąt 8 +dziewięćdziesiąt 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv new file mode 100644 index 000000000..4475ae872 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv @@ -0,0 +1,8 @@ +dwudziestu 2 +trzydziestu 3 +czterdziestu 4 +pięćdziesięciu 5 +sześćdziesięciu 6 +siedemdziesięciu 7 +osiemdziesięciu 8 +dziewięćdziesięciu 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv new file mode 100644 index 000000000..77952c803 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv @@ -0,0 +1,8 @@ +dwudziestoma 2 +trzydziestoma 3 +czterdziestoma 4 +pięćdziesięcioma 5 +sześćdziesięcioma 6 +siedemdziesięcioma 7 +osiemdziesięcioma 8 +dziewięćdziesięcioma 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv new file mode 100644 index 000000000..87e568040 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv @@ -0,0 +1,8 @@ +dwudziestka 2 +trzydziestka 3 +czterdziestka 4 +pięćdziesiątka 5 +sześćdziesiątka 6 +siedemdziesiątka 7 +osiemdziesiątka 8 +dziewięćdziesiątka 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv new file mode 100644 index 000000000..2e77b4c19 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv @@ -0,0 +1,8 @@ +dwudziesto 2 +trzydziesto 3 +czterdziesto 4 +pięćdziesięcio 5 +sześćdziesięcio 6 +siedemdziesięcio 7 +osiemdziesięcio 8 +dziewięćdziesięcio 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv new file mode 100644 index 000000000..d9d148ea6 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv @@ -0,0 +1,14 @@ +sg_nom tysiąc +sg_gen tysiąca +sg_dat tysiącowi +sg_acc tysiąc +sg_ins tysiącem +sg_loc tysiącu +sg_voc tysiącu +pl_nom tysiące +pl_gen tysięcy +pl_dat tysiącom +pl_acc tysiące +pl_ins tysiącami +pl_loc tysiącach +pl_voc tysiące diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv new file mode 100644 index 000000000..a1b116c10 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv new file mode 100644 index 000000000..85a82bd4a --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv @@ -0,0 +1,9 @@ +pierwszy 1 +drugi 2 +trzeci 3 +czwarty 4 +piąty 5 +szósty 6 +siódmy 7 +ósmy 8 +dziewiąty 9 diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv new file mode 100644 index 000000000..6044ab8e4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv @@ -0,0 +1,9 @@ +setny 1 +dwusetny 2 +trzechsetny 3 +czterechsetny 4 +pięćsetny 5 +sześćsetny 6 +siedemsetny 7 +osiemsetny 8 +dziewięćsetny 9 diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv new file mode 100644 index 000000000..07b3478ec --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv @@ -0,0 +1,10 @@ +dziesiąty 10 +jedenasty 11 +dwunasty 12 +trzynasty 13 +czternasty 14 +piętnasty 15 +szesnasty 16 +siedemnasty 17 +osiemnasty 18 +dziewiętnasty 19 diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv new file mode 100644 index 000000000..ad1795634 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv @@ -0,0 +1,8 @@ +dwudziesty 2 +trzydziesty 3 +czterdziesty 4 +pięćdziesiąty 5 +sześćdziesiąty 6 +siedemdziesiąty 7 +osiemdziesiąty 8 +dziewięćdziesiąty 9 diff --git a/nemo_text_processing/text_normalization/pl/data/roman/__init__.py b/nemo_text_processing/text_normalization/pl/data/roman/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/roman/digit.tsv b/nemo_text_processing/text_normalization/pl/data/roman/digit.tsv new file mode 100644 index 000000000..20a6ea627 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/digit.tsv @@ -0,0 +1,9 @@ +i 1 +ii 2 +iii 3 +iv 4 +v 5 +vi 6 +vii 7 +viii 8 +ix 9 diff --git a/nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv new file mode 100644 index 000000000..af2100b79 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv @@ -0,0 +1,9 @@ +c 1 +cc 2 +ccc 3 +cd 4 +d 5 +dc 6 +dcc 7 +dccc 8 +cm 9 diff --git a/nemo_text_processing/text_normalization/pl/data/roman/names.tsv b/nemo_text_processing/text_normalization/pl/data/roman/names.tsv new file mode 100644 index 000000000..1bb422766 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/names.tsv @@ -0,0 +1,12 @@ +king Ludwik noun_mp_k.tsv +king Henryk noun_mp_k.tsv +king Karol noun_mp_rz.tsv +king Filip noun_mp_p.tsv +king Jan noun_mp_p.tsv +king Ferdynand noun_mp_d.tsv +king Napoleon noun_mp_p.tsv +king Jerzy noun_mp_y.tsv +king Franciszek Józef noun_mp_ek.tsv,noun_mp_p.tsv +queen Katarzyna noun_f_na.tsv +pope Klemens noun_mp_p.tsv +pope Jan Paweł noun_mp_p.tsv,noun_mp_el.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/roman/ties.tsv b/nemo_text_processing/text_normalization/pl/data/roman/ties.tsv new file mode 100644 index 000000000..2b9e48e21 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/ties.tsv @@ -0,0 +1,9 @@ +x 1 +xx 2 +xxx 3 +xl 4 +l 5 +lx 6 +lxx 7 +lxxx 8 +xc 9 diff --git a/nemo_text_processing/text_normalization/pl/data/whitelist.tsv b/nemo_text_processing/text_normalization/pl/data/whitelist.tsv new file mode 100644 index 000000000..feb31069b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/whitelist.tsv @@ -0,0 +1,9 @@ +n.p.m. nad poziomem morza +tj. to jest +np. na przykład +itd. i tak dalej +m.in. między innymi +im. imienia +ur. urodzony +ang. angielski +tzw. tak zwany diff --git a/nemo_text_processing/text_normalization/pl/graph_utils.py b/nemo_text_processing/text_normalization/pl/graph_utils.py new file mode 100644 index 000000000..cbad45062 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/graph_utils.py @@ -0,0 +1,89 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import byte, pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space + +from .utils import get_abs_path, load_labels + +_ALPHA_UPPER = "AĄBCĆDEĘFGHIJKLŁMNŃOÓPQRSŚTUVWXYZŻŹ" +_ALPHA_LOWER = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzżź" + +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(_ALPHA_UPPER, _ALPHA_LOWER)]) +TO_UPPER = pynini.invert(TO_LOWER) + +PL_LOWER = pynini.union(*_ALPHA_LOWER).optimize() +PL_UPPER = pynini.union(*_ALPHA_UPPER).optimize() +PL_ALPHA = pynini.union(PL_LOWER, PL_UPPER).optimize() +PL_ALNUM = pynini.union(byte.DIGIT, PL_ALPHA).optimize() + +bos_or_space = pynini.union("[BOS]", " ") +eos_or_space = pynini.union("[EOS]", " ") + +ensure_space = pynini.cross(pynini.closure(delete_space, 0, 1), " ") + + +def roman_to_int(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000. + e.g. + "V" -> "5" + "i" -> "1" + + Args: + fst: Any fst. Composes fst onto Roman conversion outputs. + """ + + def _load_roman(file: str): + roman = load_labels(get_abs_path(file)) + roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman] + return pynini.string_map(roman_numerals) + + digit = _load_roman("data/roman/digit.tsv") + ties = _load_roman("data/roman/ties.tsv") + hundreds = _load_roman("data/roman/hundreds.tsv") + + graph = ( + digit + | ties + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + | ( + hundreds + + (ties | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + ).optimize() + + return graph @ fst + + +def all_to_graph(graph_dict, default=None, deterministic=False): + """ + Converts a dictionary of graphs to a single graph. + Polish has multiple cases, so this is useful for generating a single graph + """ + if default is None: + for default_key in ["mi_sg_nom", "sg_nom", "nom", "mi_pl_nom", "pl_nom"]: + if default_key in graph_dict: + break + else: + default_key = default + output_graph = graph_dict[default_key] + if not deterministic: + for key in graph_dict: + if key != default_key: + output_graph |= graph_dict[key] + return output_graph.optimize() diff --git a/nemo_text_processing/text_normalization/pl/inflection.py b/nemo_text_processing/text_normalization/pl/inflection.py new file mode 100644 index 000000000..7b1076577 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/inflection.py @@ -0,0 +1,115 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels + + +def _load_endings(grammar_file: str) -> Dict[str, str]: + return { + slot: "" if ending == "" else ending + for slot, ending in load_labels(get_abs_path(f"data/grammar/{grammar_file}")) + } + + +def inflect_noun(word: str, grammar_file: str) -> Dict[str, str]: + """Inflects a noun using a grammar file containing slot-to-ending mappings.""" + + endings = _load_endings(grammar_file) + lemma_ending = endings["sg_nom"] + if not word.endswith(lemma_ending): + raise ValueError(f"{word!r} must end in {lemma_ending!r} from {grammar_file}") + stem = word[: -len(lemma_ending)] if lemma_ending else word + return {slot: stem + ending for slot, ending in endings.items()} + + +def case_prepositions() -> Dict[str, 'pynini.FstLike']: + """Loads prepositions as case-indexed identity graphs.""" + + graphs = {} + for preposition, cases in load_labels(get_abs_path("data/grammar/prepositions.tsv")): + graph = pynini.accep(preposition) + pynutil.delete(" ") + pynutil.insert(" ") + for case in cases.split(","): + graphs[case] = graph if case not in graphs else graphs[case] | graph + return {case: graph.optimize() for case, graph in graphs.items()} + + +def inflect_abbreviation(abbreviation: str, word: str, grammar_file: str) -> Dict[str, 'pynini.FstLike']: + """Creates abbreviation-to-word graphs for every slot in a nominal paradigm.""" + + endings = _load_endings(grammar_file) + lemma_ending = endings["sg_nom"] + if not abbreviation.endswith(lemma_ending) or not word.endswith(lemma_ending): + raise ValueError(f"{abbreviation!r} and {word!r} must share the {lemma_ending!r} ending from {grammar_file}") + abbreviation_stem = abbreviation[: -len(lemma_ending)] if lemma_ending else abbreviation + word_stem = word[: -len(lemma_ending)] if lemma_ending else word + return { + slot: pynini.cross(abbreviation_stem + ending, word_stem + ending).optimize() + for slot, ending in endings.items() + } + + +def expand_abbreviation(abbreviation: str, word: str, grammar_file: str) -> Dict[str, 'pynini.FstLike']: + """Creates graphs from one ambiguous abbreviation to each singular word form.""" + + endings = _load_endings(grammar_file) + lemma_ending = endings["sg_nom"] + if not word.endswith(lemma_ending): + raise ValueError(f"{word!r} must end in {lemma_ending!r} from {grammar_file}") + word_stem = word[: -len(lemma_ending)] if lemma_ending else word + return { + slot: pynini.cross(abbreviation, word_stem + ending).optimize() + for slot, ending in endings.items() + if slot.startswith("sg_") + } + + +def load_inflected_abbreviations(filepath: str) -> Dict[str, 'pynini.FstLike']: + """Loads abbreviation, lemma, and grammar triples into slot-indexed graphs.""" + + graphs = {} + for abbreviation, word, grammar_file in load_labels(get_abs_path(filepath)): + for slot, graph in inflect_abbreviation(abbreviation, word, grammar_file).items(): + graphs[slot] = graph if slot not in graphs else graphs[slot] | graph + return {slot: graph.optimize() for slot, graph in graphs.items()} + + +def load_ambiguous_abbreviations(filepath: str) -> Dict[str, 'pynini.FstLike']: + """Loads abbreviation, lemma, and grammar triples as singular alternatives.""" + + graphs = {} + for abbreviation, word, grammar_file in load_labels(get_abs_path(filepath)): + for slot, graph in expand_abbreviation(abbreviation, word, grammar_file).items(): + graphs[slot] = graph if slot not in graphs else graphs[slot] | graph + return {slot: graph.optimize() for slot, graph in graphs.items()} + + +def load_adjective_abbreviations(filepath: str) -> Dict[str, 'pynini.FstLike']: + """Loads abbreviations whose adjective component exposes every inflectional slot.""" + + from nemo_text_processing.text_normalization.pl.taggers.ordinal import complete_paradigm + + graphs = {} + for abbreviation, prefix, adjective in load_labels(get_abs_path(filepath)): + forms = adjective_inflection(adjective) + complete_paradigm(forms, complete=True) + for slot, form in forms.items(): + spoken = f"{prefix} {form}" if prefix else form + graph = pynini.cross(abbreviation, spoken) + graphs[slot] = graph if slot not in graphs else graphs[slot] | graph + return {slot: graph.optimize() for slot, graph in graphs.items()} diff --git a/nemo_text_processing/text_normalization/pl/licence b/nemo_text_processing/text_normalization/pl/licence new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/licence @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/taggers/__init__.py b/nemo_text_processing/text_normalization/pl/taggers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py b/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py new file mode 100644 index 000000000..9cc8eac7d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.pl.graph_utils import PL_UPPER + + +class AbbreviationFst(GraphFst): + """Classifies uppercase Polish initialisms, with or without dots.""" + + def __init__(self, whitelist=None, deterministic: bool = True): + super().__init__(name="abbreviation", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + graph = PL_UPPER + dot + pynini.closure(insert_space + PL_UPPER + dot, 1) + graph |= PL_UPPER + dot + pynini.closure(PL_UPPER + dot, 1) + graph |= PL_UPPER + pynini.closure(insert_space + PL_UPPER, 1) + + if whitelist is not None: + graph = pynini.compose( + pynini.difference(pynini.project(graph, "input"), pynini.project(whitelist.graph, "input")), graph + ) + + graph = pynutil.insert('value: "') + graph.optimize() + pynutil.insert('"') + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py new file mode 100644 index 000000000..5467262a9 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -0,0 +1,347 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022, 2023 Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Iterable + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, delete_space +from nemo_text_processing.text_normalization.pl.graph_utils import PL_ALPHA +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels + +CASES = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] +DEFAULT_SLOT = "mi_sg_nom" +SCALE_NAMES = ["tysiąc", "milion", "miliard", "bilion", "biliard", "trylion", "tryliard"] + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + exactly_three_digits = NEMO_DIGIT**3 + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) + cardinal_string = pynini.closure(NEMO_DIGIT, 1) + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(" ") + + pynini.closure(exactly_three_digits + pynutil.delete(" ")) + + exactly_three_digits + ) + return cardinal_string @ fst + + +def get_digit_forms(filepath: str) -> Dict[str, Dict[str, object]]: + output = {} + for digit, grammar, form in load_labels(get_abs_path(filepath)): + forms = output.setdefault(digit, {}) + if grammar not in forms: + forms[grammar] = form + elif isinstance(forms[grammar], list): + forms[grammar].append(form) + else: + forms[grammar] = [forms[grammar], form] + return output + + +def _forms_to_graphs( + forms: Dict[str, Dict[str, object]], deterministic: bool +) -> Dict[str, Dict[str, 'pynini.FstLike']]: + graphs = {} + for number, slots in forms.items(): + graphs[number] = {} + for slot, values in slots.items(): + values = values if isinstance(values, list) else [values] + if deterministic: + values = values[:1] + graphs[number][slot] = pynini.union(*(pynini.cross(number, value) for value in values)).optimize() + return graphs + + +def _invert_string_file(path: str) -> 'pynini.FstLike': + return pynini.invert(pynini.string_file(get_abs_path(path))).optimize() + + +def _case_for_slot(slot: str) -> str: + if slot == "compound": + return slot + for case in CASES: + if slot == case or slot.endswith(f"_{case}"): + return case + raise ValueError(f"Cannot determine case from slot: {slot}") + + +def _select(mapping: Dict[str, 'pynini.FstLike'], keys: Iterable[str]) -> 'pynini.FstLike': + for key in keys: + if key in mapping: + return mapping[key] + raise KeyError(f"None of {list(keys)} is available") + + +def _noun_forms(lemma: str) -> Dict[str, str]: + if lemma == "tysiąc": + return {key: value for key, value in load_labels(get_abs_path("data/numbers/tysiac.tsv"))} + + stem = lemma + loc_sg = "u" + if lemma.endswith("ion"): + loc_sg = "ie" + elif lemma.endswith("iard"): + loc_sg = "zie" + return { + "sg_nom": lemma, + "sg_gen": lemma + "a", + "sg_dat": lemma + "owi", + "sg_acc": lemma, + "sg_ins": lemma + "em", + "sg_loc": stem + loc_sg, + "sg_voc": lemma + "ie", + "pl_nom": lemma + "y", + "pl_gen": lemma + "ów", + "pl_dat": lemma + "om", + "pl_acc": lemma + "y", + "pl_ins": lemma + "ami", + "pl_loc": lemma + "ach", + "pl_voc": lemma + "y", + } + + +class CardinalFst(GraphFst): + """Classifies Polish cardinal numbers and exposes each inflectional graph in ``graphs``.""" + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit_forms = get_digit_forms("data/numbers/digit_forms.tsv") + teen_forms = get_digit_forms("data/numbers/teens_forms.tsv") + digit_graphs = _forms_to_graphs(digit_forms, deterministic) + teen_graphs = _forms_to_graphs(teen_forms, deterministic) + + jeden = adjective_inflection("jeden", compound="jedno") + from nemo_text_processing.text_normalization.pl.taggers.ordinal import complete_paradigm + + complete_paradigm(jeden, complete=True) + self.jeden_all = {slot: pynini.cross("1", form) for slot, form in jeden.items()} + + zero_forms = { + "sg_nom": "zero", + "sg_gen": "zera", + "sg_dat": "zeru", + "sg_acc": "zero", + "sg_ins": "zerem", + "sg_loc": "zerze", + "sg_voc": "zero", + } + self.zero_all = {slot: pynini.cross("0", form) for slot, form in zero_forms.items()} + self.zero_sg = {slot[3:]: graph for slot, graph in self.zero_all.items()} + + ordinary_slots = set(self.jeden_all) + for forms in digit_forms.values(): + ordinary_slots.update(forms) + for forms in teen_forms.values(): + ordinary_slots.update(forms) + + tens_nom = _invert_string_file("data/numbers/tens.tsv") + tens_gen = _invert_string_file("data/numbers/tens_gen.tsv") + tens_ins = _invert_string_file("data/numbers/tens_ins.tsv") + tens_compound = _invert_string_file("data/numbers/tens_prefix.tsv") + hundreds_nom = _invert_string_file("data/numbers/hundreds.tsv") + hundreds_gen = _invert_string_file("data/numbers/hundreds_gen.tsv") + hundreds_ins = _invert_string_file("data/numbers/hundreds_ins.tsv") + hundreds_compound = _invert_string_file("data/numbers/hundreds.tsv") + + join = pynutil.insert(" ") + compound_join = pynutil.insert("") + if not deterministic: + compound_join |= pynutil.add_weight(pynutil.insert(" "), 0.001) + + self.graphs = {} + self.two_digit_graphs = {} + self.hundreds_graphs = {} + + for slot in sorted(ordinary_slots): + case = _case_for_slot(slot) + if case == "compound": + tens = tens_compound + hundreds = hundreds_compound + component_join = compound_join + elif case == "ins": + tens = tens_ins + hundreds = hundreds_ins + component_join = join + elif case in {"gen", "dat", "loc"} or slot.startswith("mp_"): + tens = tens_gen + hundreds = hundreds_gen + component_join = join + else: + tens = tens_nom + hundreds = hundreds_nom + component_join = join + + digit = self._digit_for_slot(digit_graphs, slot) + compound_digit = self._compound_digit_for_slot(digit_graphs, slot) + teen = self._teen_for_slot(teen_graphs, slot) + isolated_one = self._one_for_slot(slot, compound=False, deterministic=deterministic) + compound_one = self._one_for_slot(slot, compound=True, deterministic=deterministic) + two_digit = ( + teen + | tens + pynutil.delete("0") + | tens + component_join + (compound_digit | compound_one) + | pynutil.delete("0") + (digit | isolated_one) + ).optimize() + hundred = ( + hundreds + pynutil.delete("00") + | hundreds + component_join + two_digit + | pynutil.delete("0") + two_digit + | pynutil.delete("00") + (digit | isolated_one) + ).optimize() + + self.two_digit_graphs[slot] = two_digit + self.hundreds_graphs[slot] = hundred + for slot, hundred in self.hundreds_graphs.items(): + self.graphs[slot] = self._make_full_number_graph(hundred, slot, deterministic) + + self.graph_dict = self.graphs + compound_boundary = pynutil.delete("-") + if not deterministic: + compound_boundary += pynini.union(pynutil.insert(""), pynutil.add_weight(pynutil.insert(" "), 0.001)) + self.compound = (self.graphs["compound"] + compound_boundary + pynini.closure(PL_ALPHA, 1)).optimize() + + self.graph = filter_punctuation(self.graphs[DEFAULT_SLOT] | self.zero_all["sg_nom"]).optimize() | self.compound + if not deterministic: + self.graph = ( + filter_punctuation(pynini.union(*self.graphs.values(), *self.zero_all.values())).optimize() + | self.compound + ) + + self.graph_unfiltered = self.graph + optional_minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + final_graph = optional_minus + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + self.fst = self.add_tokens(final_graph).optimize() + + def _digit_for_slot(self, graphs, slot): + choices = { + "f_pl_nom": ["f_pl_nom", "mi_pl_nom"], + "f_pl_ins": ["f_pl_ins", "pl_ins"], + "mi_pl_ins": ["mi_pl_ins", "pl_ins"], + "pl_ins": ["mi_pl_ins", "pl_ins"], + }.get(slot, [slot]) + case = _case_for_slot(slot) + if case in {"acc", "voc"}: + choices += [slot.rsplit("_", 1)[0] + "_nom", "mi_pl_nom"] + choices += [f"pl_{case}", "mi_pl_nom"] + return pynini.union(*(_select(forms, choices) for forms in graphs.values())).optimize() + + def _compound_digit_for_slot(self, graphs, slot): + if slot.startswith("mp_") and _case_for_slot(slot) in {"nom", "acc"}: + choices = ["pl_gen", "mp_pl_nom"] + elif slot == "pl_ins": + choices = ["mi_pl_ins", "pl_ins"] + else: + choices = [slot] + case = _case_for_slot(slot) + choices += [f"pl_{case}", "mi_pl_nom"] + return pynini.union(*(_select(forms, choices) for forms in graphs.values())).optimize() + + def _teen_for_slot(self, graphs, slot): + case = _case_for_slot(slot) + choices = [slot, f"pl_{case}"] + if case in {"acc", "voc"}: + choices += ["mp_pl_nom" if slot.startswith("mp_") else "mi_pl_nom"] + choices += ["mi_pl_nom"] + return pynini.union(*(_select(forms, choices) for forms in graphs.values())).optimize() + + def _one_for_slot(self, slot, compound, deterministic): + if slot == "compound": + return self.jeden_all[slot] + if not compound: + return self.jeden_all[slot] + case = _case_for_slot(slot) + graph = pynini.cross("1", "jeden") + if not deterministic: + key = slot if slot in self.jeden_all else f"mi_sg_{case}" + graph |= pynutil.add_weight(self.jeden_all[key], 0.001) + return graph.optimize() + + def _make_full_number_graph(self, group, slot, deterministic): + case = _case_for_slot(slot) + if case == "compound": + short_input = pynini.closure(NEMO_DIGIT, 1, 3) + pad = ( + short_input + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**3 + ) + return (pad @ group).optimize() + + scale_slot = { + "nom": "mi_sg_nom", + "acc": "mi_sg_acc", + "voc": "mi_sg_voc", + "gen": "pl_gen", + "dat": "pl_dat", + "ins": "mi_pl_ins", + "loc": "pl_gen", + }[case] + scale_group = self.hundreds_graphs[scale_slot] + + plural_group = self._restrict_group(scale_group, "plural") + quantity_group = self._restrict_group(scale_group, "quantity") + non_one_group = self._restrict_group(scale_group, "non_one") + factors = [] + for scale in reversed(SCALE_NAMES): + forms = _noun_forms(scale) + if case in {"nom", "acc", "voc"}: + singular = forms[f"sg_{case}"] + plural = forms[f"pl_{case}"] + quantity = forms["pl_gen"] + factor = ( + pynutil.delete("000") + | pynini.cross("001", singular) + pynutil.insert(" ") + | plural_group + pynutil.insert(" " + plural + " ") + | quantity_group + pynutil.insert(" " + quantity + " ") + ) + else: + factor = ( + pynutil.delete("000") + | pynini.cross("001", forms[f"sg_{case}"]) + pynutil.insert(" ") + | non_one_group + pynutil.insert(" " + forms[f"pl_{case}"] + " ") + ) + if not deterministic: + factor |= pynutil.add_weight(pynini.cross("001", "jeden " + forms[f"sg_{case}"] + " "), 0.001) + factors.append(factor) + + padded = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + ) + full = None + for factor in factors: + full = factor if full is None else full + factor + full += group | pynutil.delete("000") + clean = pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + return (padded @ full @ clean).optimize() + + @staticmethod + def _restrict_group(group, kind): + hundred = NEMO_DIGIT + if kind == "plural": + inputs = hundred + (NEMO_DIGIT - "1") + pynini.union("2", "3", "4") + elif kind == "quantity": + inputs = hundred + pynini.union( + "1" + NEMO_DIGIT, + (NEMO_DIGIT - "1") + pynini.union("0", "5", "6", "7", "8", "9"), + (NEMO_DIGIT - "0") + "1", + ) + else: + inputs = NEMO_DIGIT**3 - "001" - "000" + return inputs @ group diff --git a/nemo_text_processing/text_normalization/pl/taggers/date.py b/nemo_text_processing/text_normalization/pl/taggers/date.py new file mode 100644 index 000000000..79678f0b0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/date.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.pl.utils import get_abs_path + + +class DateFst(GraphFst): + """Classifies Polish dates and exposes case-coordinated graphs.""" + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + day_input = pynini.union(*(str(day) for day in range(1, 32))) + numeric_day_input = day_input | pynini.union(*(f"{day:02d}" for day in range(1, 10))) + + month_number = pynini.string_file(get_abs_path("data/dates/months.tsv")) + month_number = pynutil.delete("0") + month_number | month_number + month_words = pynini.project(month_number, "output") + month_abbr = pynini.string_file(get_abs_path("data/dates/month_abbr.tsv")) + month_roman = pynini.string_file(get_abs_path("data/dates/months_roman.tsv")) + + year_prefix = ((NEMO_DIGIT - "0") + pynutil.insert("000")) @ cardinal.graphs["mi_sg_nom"] + + month_numeric_field = pynutil.insert(' month: "') + month_number + pynutil.insert('"') + month_roman_field = pynutil.insert(' month: "') + month_roman + pynutil.insert('"') + month_word_field = pynutil.insert(' month: "') + (month_words | month_abbr) + pynutil.insert('"') + + self.graphs = {} + self.year_graphs = {} + for slot, ordinal_graph in ordinal.graphs.items(): + if slot == "compound": + continue + day = day_input @ ordinal_graph + numeric_day = numeric_day_input @ ordinal_graph + year = year_prefix + insert_space + (NEMO_DIGIT**3 @ ordinal_graph) + year_with_abbreviation = year + pynini.closure( + pynini.closure(delete_space, 0, 1) + pynini.cross("r.", " roku"), 0, 1 + ) + day_field = pynutil.insert('day: "') + day + pynutil.insert('"') + numeric_day_field = pynutil.insert('day: "') + numeric_day + pynutil.insert('"') + year_field = pynutil.insert(' year: "') + year_with_abbreviation + pynutil.insert('"') + + numeric = pynini.union( + *( + numeric_day_field + + pynutil.delete(separator) + + month_numeric_field + + pynutil.delete(separator) + + year_field + for separator in (".", "-", "/") + ) + ) + numeric |= numeric_day_field + pynutil.delete(".") + month_roman_field + pynutil.delete(".") + year_field + written = day_field + delete_space + month_word_field + written += pynini.closure(delete_space + year_field, 0, 1) + self.graphs[slot] = (numeric | written).optimize() + self.year_graphs[slot] = (pynutil.insert('year: "') + year + pynutil.insert('"')).optimize() + + self.graph_dict = self.graphs + if deterministic: + self.final_graph = self.graphs["mi_sg_gen"] + else: + self.final_graph = pynini.union(*self.graphs.values(), *self.year_graphs.values()).optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/measure.py b/nemo_text_processing/text_normalization/pl/taggers/measure.py new file mode 100644 index 000000000..95bc07ef0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/measure.py @@ -0,0 +1,73 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.text_normalization.pl.inflection import inflect_noun +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels + + +def _case(slot: str) -> str: + for case in ("nom", "gen", "dat", "acc", "ins", "loc", "voc"): + if slot == case or slot.endswith(f"_{case}"): + return case + raise ValueError(f"Cannot determine case from {slot!r}") + + +class MeasureFst(GraphFst): + """Classifies integer measures with case-inflected masculine units.""" + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + unit_graphs = {} + for symbol, lemma, grammar_file in load_labels(get_abs_path("data/measures/units.tsv")): + for slot, form in inflect_noun(lemma, grammar_file).items(): + graph = pynini.cross(symbol, form) + unit_graphs[slot] = graph if slot not in unit_graphs else unit_graphs[slot] | graph + unit_graphs = {slot: graph.optimize() for slot, graph in unit_graphs.items()} + + positive = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) + one = pynini.accep("1") + few = pynini.intersect(positive, pynini.closure(NEMO_DIGIT) + pynini.union("2", "3", "4")) + few = pynini.difference(few, pynini.closure(NEMO_DIGIT) + pynini.union("12", "13", "14")) + many = pynini.union("0", pynini.difference(pynini.difference(positive, one), few)).optimize() + optional_space = pynini.closure(delete_space, 0, 1) + + def graph_for(number_input, number_graph, unit_slot): + return ( + pynutil.insert('cardinal { integer: "') + + (number_input @ number_graph) + + pynutil.insert('" } units: "') + + optional_space + + unit_graphs[unit_slot] + + pynutil.insert('"') + ) + + self.graphs = {} + for slot, number_graph in cardinal.graphs.items(): + if slot == "compound": + continue + case = _case(slot) + graph = graph_for(one, number_graph, f"sg_{case}") + graph |= graph_for(few, number_graph, f"pl_{case}") + graph |= graph_for(many, number_graph, "pl_gen") + self.graphs[slot] = graph.optimize() + + self.graph_dict = self.graphs + graph = self.graphs["mi_sg_nom"] if deterministic else pynini.union(*self.graphs.values()) + self.final_graph = graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py new file mode 100644 index 000000000..fc8c94007 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -0,0 +1,118 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space +from nemo_text_processing.text_normalization.pl.graph_utils import all_to_graph +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels + + +def complete_paradigm(partial: Dict[str, str], complete: bool = False): + partial["mi_sg_acc"] = partial["mi_sg_nom"] + partial["mi_sg_loc"] = partial["mi_sg_ins"] + partial["mi_sg_voc"] = partial["mi_sg_nom"] + partial["ma_sg_nom"] = partial["mi_sg_nom"] + partial["ma_sg_gen"] = partial["mi_sg_gen"] + partial["ma_sg_dat"] = partial["mi_sg_dat"] + partial["ma_sg_acc"] = partial["mi_sg_gen"] + partial["ma_sg_ins"] = partial["mi_sg_ins"] + partial["ma_sg_loc"] = partial["mi_sg_loc"] + partial["ma_sg_voc"] = partial["mi_sg_voc"] + for case in ["nom", "gen", "dat", "acc", "ins", "loc", "voc"]: + partial[f"mp_sg_{case}"] = partial[f"ma_sg_{case}"] + partial["nt_sg_gen"] = partial["mi_sg_gen"] + partial["nt_sg_dat"] = partial["mi_sg_dat"] + partial["nt_sg_acc"] = partial["nt_sg_nom"] + partial["nt_sg_ins"] = partial["mi_sg_ins"] + partial["nt_sg_loc"] = partial["mi_sg_loc"] + partial["nt_sg_voc"] = partial["nt_sg_nom"] + partial["f_sg_dat"] = partial["f_sg_gen"] + partial["f_sg_acc"] = partial["f_sg_ins"] + partial["f_sg_loc"] = partial["f_sg_gen"] + partial["f_sg_voc"] = partial["f_sg_nom"] + partial["mp_pl_acc"] = partial["pl_loc"] + partial["mp_pl_voc"] = partial["mp_pl_nom"] + partial["pl_nom"] = partial["nt_sg_nom"] + partial["pl_gen"] = partial["pl_loc"] + partial["pl_dat"] = partial["mi_sg_ins"] + partial["pl_acc"] = partial["pl_nom"] + partial["pl_voc"] = partial["pl_nom"] + if complete: + for gender in ["mi", "ma", "mp", "nt", "f"]: + for case in ["nom", "gen", "dat", "acc", "ins", "loc", "voc"]: + key = f"{gender}_pl_{case}" + if key not in partial: + partial[key] = partial[f"pl_{case}"] + + +def make_graph_dict(filepath: str, invert: bool = True, complete: bool = False): + output = {} + for word, target in load_labels(get_abs_path(filepath)): + forms = adjective_inflection(word) + if complete: + complete_paradigm(forms, complete=True) + for slot, form in forms.items(): + source, destination = (target, form) if invert else (form, target) + graph = pynini.cross(source, destination) + output[slot] = graph if slot not in output else output[slot] | graph + return {slot: graph.optimize() for slot, graph in output.items()} + + +class OrdinalFst(GraphFst): + """Classifies Polish ordinals and exposes adjective-inflected graphs in ``graphs``.""" + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + digits = make_graph_dict("data/ordinal/digit.tsv", complete=True) + tens = make_graph_dict("data/ordinal/tens.tsv", complete=True) + teens = make_graph_dict("data/ordinal/teens.tsv", complete=True) + hundreds = make_graph_dict("data/ordinal/hundreds.tsv", complete=True) + cardinal_hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))).optimize() + + self.graphs = {} + for slot in digits: + joiner = pynutil.insert("") if slot == "compound" else insert_space + if slot == "compound" and not deterministic: + joiner |= pynutil.add_weight(insert_space, 0.001) + + two_digit = ( + tens[slot] + pynutil.delete("0") + | pynutil.delete("0") + digits[slot] + | teens[slot] + | tens[slot] + joiner + digits[slot] + ).optimize() + three_digit = ( + hundreds[slot] + pynutil.delete("00") + | pynutil.delete("0") + two_digit + | cardinal_hundreds + joiner + two_digit + ).optimize() + short_input = pynini.closure(NEMO_DIGIT, 1, 3) + pad = ( + short_input + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**3 + ) + self.graphs[slot] = (pad @ three_digit).optimize() + + self.graph_dict = self.graphs + graph = all_to_graph(self.graphs, deterministic=deterministic) + if not deterministic: + graph = pynini.union(*self.graphs.values()).optimize() + self.graph = (graph + pynutil.delete(".")).optimize() + final_graph = pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/roman.py b/nemo_text_processing/text_normalization/pl/taggers/roman.py new file mode 100644 index 000000000..cd21d440f --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/roman.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, delete_space, insert_space +from nemo_text_processing.text_normalization.pl.graph_utils import roman_to_int +from nemo_text_processing.text_normalization.pl.inflection import case_prepositions, inflect_noun +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels + + +def _name_forms(name: str, grammar_files: str): + components = name.split(" ") + grammars = grammar_files.split(",") + if len(components) != len(grammars): + raise ValueError(f"{name!r} must have one grammar per component") + paradigms = [inflect_noun(component, grammar) for component, grammar in zip(components, grammars)] + slots = set.intersection(*(set(paradigm) for paradigm in paradigms)) + return {slot: " ".join(paradigm[slot] for paradigm in paradigms) for slot in slots} + + +class RomanFst(GraphFst): + """Classifies Roman ordinals following curated ruler and papal names.""" + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + prepositions = case_prepositions() + self.graphs = {} + for category, name, grammar_files in load_labels(get_abs_path("data/roman/names.tsv")): + gender = "f" if category == "queen" else "mp" + for noun_slot, surface_name in _name_forms(name, grammar_files).items(): + number, case = noun_slot.split("_", 1) + ordinal_slot = f"{gender}_{number}_{case}" + if ordinal_slot not in ordinal.graphs: + continue + name_graph = pynini.accep(surface_name) + if category == "pope" and noun_slot == "sg_nom": + title = pynini.union("Papież", "papież") + delete_space + insert_space + name_graph |= title + pynini.accep(surface_name) + graph = name_graph + delete_space + insert_space + roman_to_int(ordinal.graphs[ordinal_slot]) + if case in prepositions: + graph |= prepositions[case] + graph + self.graphs[ordinal_slot] = ( + graph if ordinal_slot not in self.graphs else self.graphs[ordinal_slot] | graph + ) + + self.graph_dict = {slot: graph.optimize() for slot, graph in self.graphs.items()} + graph = pynini.union(*self.graph_dict.values()).optimize() + self.graph = graph + final_graph = pynutil.insert('name: "') + convert_space(graph) + pynutil.insert('"') + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/time.py b/nemo_text_processing/text_normalization/pl/taggers/time.py new file mode 100644 index 000000000..90fff1de4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/time.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space + + +class TimeFst(GraphFst): + """Classifies Polish numeric hours and minutes.""" + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + hour_numbers = pynini.union(*(str(hour) for hour in range(1, 24))) + hours = hour_numbers | pynutil.delete("0") + pynini.union(*(str(hour) for hour in range(1, 10))) + minutes = pynini.union(*(f"{minute:02d}" for minute in range(1, 60))) + minute_words = (pynutil.delete("0") + cardinal.graphs["mi_sg_nom"]) | cardinal.graphs["mi_sg_nom"] + + def time_graph(hour_slot: str, prefix: 'pynini.FstLike') -> 'pynini.FstLike': + hour = hours @ ordinal.graphs[hour_slot] + hour_field = pynutil.insert('hours: "') + prefix + hour + pynutil.insert('"') + minute_field = pynutil.insert(' minutes: "') + (minutes @ minute_words) + pynutil.insert('"') + separator = pynutil.delete(pynini.union(":", ".")) + return hour_field + separator + (pynutil.delete("00") | minute_field) + + plain = time_graph("f_sg_nom", pynini.accep("")) + governed = time_graph("f_sg_loc", pynini.accep("o") + delete_space + pynutil.insert(" ")) + hour_abbreviation = ( + pynini.accep("o") + + delete_space + + pynutil.insert(" ") + + pynini.cross("godz.", "godzinie") + + delete_space + + pynutil.insert(" ") + ) + governed |= time_graph("f_sg_loc", hour_abbreviation) + + locale_hour = pynini.cross("00", "zero") | hours @ ordinal.graphs["f_sg_nom"] + locale_minute = pynini.cross("00", "zero") | minutes @ minute_words + locale_second = pynini.cross("00", "zero") | minutes @ minute_words + locale_time = ( + pynutil.insert('hours: "') + + locale_hour + + pynutil.insert('"') + + pynutil.delete(":") + + pynutil.insert(' minutes: "') + + locale_minute + + pynutil.insert('"') + + pynutil.delete(":") + + pynutil.insert(' seconds: "') + + locale_second + + pynutil.insert('"') + ) + self.final_graph = (plain | governed | locale_time).optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..12b1c8ecd --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py @@ -0,0 +1,83 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.pl.taggers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.taggers.date import DateFst +from nemo_text_processing.text_normalization.pl.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.pl.taggers.time import TimeFst +from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.pl.taggers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"pl_tn_{deterministic}_{input_case}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + return + + self.cardinal = CardinalFst(deterministic=deterministic) + self.ordinal = OrdinalFst(deterministic=deterministic) + self.roman = RomanFst(self.ordinal, deterministic=deterministic) + self.date = DateFst(self.cardinal, self.ordinal, deterministic=deterministic) + self.measure = MeasureFst(self.cardinal, deterministic=deterministic) + self.time = TimeFst(self.cardinal, self.ordinal, deterministic=deterministic) + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + word = WordFst(deterministic=deterministic).fst + punctuation = PunctuationFst(deterministic=deterministic).fst + classify = ( + pynutil.add_weight(self.whitelist.fst, 1.01) + | pynutil.add_weight(self.roman.fst, 1.02) + | pynutil.add_weight(self.date.fst, 1.05) + | pynutil.add_weight(self.time.fst, 1.05) + | pynutil.add_weight(self.measure.fst, 1.06) + | pynutil.add_weight(self.ordinal.fst, 1.09) + | pynutil.add_weight(self.cardinal.fst, 1.1) + | pynutil.add_weight(punctuation, 2.1) + | pynutil.add_weight(word, 100) + ) + if not deterministic: + classify |= pynutil.add_weight(AbbreviationFst(whitelist=self.whitelist, deterministic=False).fst, 100) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + graph = delete_space + token + pynini.closure(delete_extra_space + token) + delete_space + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammar was saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py new file mode 100644 index 000000000..cc4177c67 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py @@ -0,0 +1,152 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Optional + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst as vAbbreviationFst +from nemo_text_processing.text_normalization.pl.taggers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.taggers.date import DateFst +from nemo_text_processing.text_normalization.pl.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.pl.taggers.time import TimeFst +from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.pl.verbalizers.cardinal import CardinalFst as vCardinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.date import DateFst as vDateFst +from nemo_text_processing.text_normalization.pl.verbalizers.measure import MeasureFst as vMeasureFst +from nemo_text_processing.text_normalization.pl.verbalizers.ordinal import OrdinalFst as vOrdinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.roman import RomanFst as vRomanFst +from nemo_text_processing.text_normalization.pl.verbalizers.time import TimeFst as vTimeFst + + +class ClassifyFst(GraphFst): + """Composes Polish classification and verbalization for audio-based TN.""" + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = True, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"_{input_case}_pl_tn_{deterministic}_with_audio.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + else: + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + roman = RomanFst(ordinal, deterministic=deterministic) + date = DateFst(cardinal, ordinal, deterministic=deterministic) + measure = MeasureFst(cardinal, deterministic=deterministic) + time = TimeFst(cardinal, ordinal, deterministic=deterministic) + whitelist_graph = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + v_cardinal = vCardinalFst(deterministic=deterministic) + v_ordinal = vOrdinalFst(deterministic=deterministic) + v_roman = vRomanFst(deterministic=deterministic) + v_date = vDateFst(deterministic=deterministic) + v_measure = vMeasureFst(deterministic=deterministic) + v_time = vTimeFst(deterministic=deterministic) + word = pynini.closure(NEMO_NOT_SPACE, 1) + punctuation = PunctuationFst(deterministic=True).graph + + sem_w = 1 + word_w = 100 + punct_w = 2 + classify_and_verbalize = ( + pynutil.add_weight(whitelist_graph.graph, sem_w) + | pynutil.add_weight(pynini.compose(roman.fst, v_roman.fst), sem_w) + | pynutil.add_weight(pynini.compose(date.fst, v_date.fst), sem_w) + | pynutil.add_weight(pynini.compose(measure.fst, v_measure.fst), sem_w) + | pynutil.add_weight(pynini.compose(time.fst, v_time.fst), sem_w) + | pynutil.add_weight(pynini.compose(cardinal.fst, v_cardinal.fst), sem_w) + | pynutil.add_weight(pynini.compose(ordinal.fst, v_ordinal.fst), sem_w) + | pynutil.add_weight(word, word_w) + ).optimize() + if not deterministic: + abbreviation = AbbreviationFst(whitelist=whitelist_graph, deterministic=False) + v_abbreviation = vAbbreviationFst(deterministic=False) + classify_and_verbalize |= pynutil.add_weight( + pynini.compose(abbreviation.fst, v_abbreviation.fst), word_w + ) + punct_only = pynutil.add_weight(punctuation, punct_w) + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | pynutil.insert(" ") + punct_only, + 1, + ) + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + classify_and_verbalize + + pynini.closure(pynutil.insert(" ") + punct) + ) + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | pynutil.insert(" ") + punct + pynutil.insert(" ") + ) + + token_plus_punct + ) + graph |= punct_only + pynini.closure(punct) + graph = delete_space + graph + delete_space + remove_extra_spaces = pynini.closure(NEMO_NOT_SPACE, 1) + pynini.closure( + delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1) + ) + remove_extra_spaces |= ( + pynini.closure(pynutil.delete(" "), 1) + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynini.closure(delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1)) + ) + self.fst = pynini.compose(graph.optimize(), remove_extra_spaces).optimize() + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + def lattice(self, text: str) -> 'pynini.Fst': + lattice = pynini.compose(pynini.accep(pynini.escape(text)), self.fst) + if lattice.start() == pynini.NO_STATE_ID: + raise ValueError(f"Polish TN failed for input: {text}") + return lattice + + def normalize(self, text: str, lm: Optional['pynini.FstLike'] = None) -> str: + lattice = self.lattice(text) + if lm is not None: + lattice = pynini.compose(lattice, lm) + if lattice.start() == pynini.NO_STATE_ID: + raise ValueError("The language model rejected every Polish TN path") + best = pynini.shortestpath(lattice, nshortest=1, unique=True).project("output") + return best.string() diff --git a/nemo_text_processing/text_normalization/pl/taggers/whitelist.py b/nemo_text_processing/text_normalization/pl/taggers/whitelist.py new file mode 100644 index 000000000..7c68b4555 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/whitelist.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from nemo_text_processing.text_normalization.pl.inflection import ( + load_adjective_abbreviations, + load_ambiguous_abbreviations, + load_inflected_abbreviations, +) +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels + + +def _get_whitelist_graph(input_case: str, filepath: str) -> 'pynini.FstLike': + labels = load_labels(filepath) + if input_case == "lower_cased": + labels = [[entry[0].lower()] + entry[1:] for entry in labels] + return pynini.string_map(labels).optimize() + + +class WhiteListFst(GraphFst): + """Classifies fixed and productively inflected Polish abbreviations.""" + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), 0.0001 + ) + + self.inflected_graphs = load_inflected_abbreviations("data/abbreviations.tsv") + graph |= pynini.union(*self.inflected_graphs.values()) + + self.nondeterministic_graphs = load_ambiguous_abbreviations("data/abbreviations_nondet.tsv") + self.adjective_graphs = load_adjective_abbreviations("data/abbreviations_adjective_nondet.tsv") + if not deterministic: + graph |= pynini.union(*self.nondeterministic_graphs.values(), *self.adjective_graphs.values()) + + if input_file: + provided = _get_whitelist_graph(input_case, input_file) + graph = graph | provided if not deterministic else provided + + self.graph = graph.optimize() + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert('name: "') + self.final_graph + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/word.py b/nemo_text_processing/text_normalization/pl/taggers/word.py new file mode 100644 index 000000000..9c0cdb7c7 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/word.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + self.fst = (pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/pl/utils.py b/nemo_text_processing/text_normalization/pl/utils.py new file mode 100644 index 000000000..6ba2a6e12 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/utils.py @@ -0,0 +1,119 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def adjective_inflection(word: str, compound: str = "") -> dict: + """ + inflect adjectives based on their endings. + This includes things like ordinals and 'jeden' (1) which inflect like adjectives. + """ + + def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): + if stem_b == "": + stem_b = stem + if compound == "": + compound = stem_b + "o" + return { + "mi_sg_nom": mi_sg, + "mi_sg_gen": stem + "ego", + "mi_sg_dat": stem + "emu", + "mi_sg_ins": stem + vowel + "m", + "nt_sg_nom": stem + "e", + "f_sg_nom": stem_b + "a", + "f_sg_gen": stem + "ej", + "f_sg_ins": stem_b + "ą", + "mp_pl_nom": mp_pl, + "pl_ins": stem + vowel + "mi", + "pl_loc": stem + vowel + "ch", + "compound": compound, + } + + stem_b = "" + if word.endswith("en"): + stem = word[:-2] + "n" + mi_sg = word + mp_pl = stem + "i" + vowel = "y" + elif word[-2:] in ["ni", "ci"]: + stem = word + mi_sg = word + mp_pl = word + vowel = "" + elif word.endswith("ony"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-3] + "eni" + vowel = "y" + elif word.endswith("szy"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "i" + vowel = "y" + elif word.endswith("gi"): + stem = word + stem_b = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "dzy" + vowel = "" + elif word.endswith("ki"): + stem = word + stem_b = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "cy" + vowel = "" + elif word.endswith("sty"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-3] + "ści" + vowel = "y" + elif word.endswith("ty"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "ci" + vowel = "y" + elif word.endswith("y"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-1] + "i" + vowel = "y" + return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/__init__.py b/nemo_text_processing/text_normalization/pl/verbalizers/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py new file mode 100644 index 000000000..3120f7261 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + sign = pynini.closure(pynini.cross('negative: "true"', "minus") + delete_space, 0, 1) + integer = pynutil.delete("integer:") + delete_space + pynutil.delete('"') + integer += pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + self.fst = self.delete_tokens(sign + integer).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/date.py b/nemo_text_processing/text_normalization/pl/verbalizers/date.py new file mode 100644 index 000000000..58ba637d4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/date.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DateFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + value = pynini.closure(NEMO_NOT_QUOTE, 1) + day = pynutil.delete('day: "') + value + pynutil.delete('"') + month = pynutil.delete('month: "') + value + pynutil.delete('"') + year = pynutil.delete('year: "') + value + pynutil.delete('"') + graph = day + delete_space + pynutil.insert(" ") + month + graph += pynini.closure(delete_space + pynutil.insert(" ") + year, 0, 1) + self.fst = self.delete_tokens(graph | year).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/measure.py b/nemo_text_processing/text_normalization/pl/verbalizers/measure.py new file mode 100644 index 000000000..ae4d6f13a --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/measure.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class MeasureFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + value = pynini.closure(NEMO_NOT_QUOTE, 1) + cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + pynutil.delete('integer: "') + + value + + pynutil.delete('"') + + delete_space + + pynutil.delete("}") + ) + units = pynutil.delete('units: "') + value + pynutil.delete('"') + self.fst = self.delete_tokens(cardinal + delete_space + pynutil.insert(" ") + units).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py new file mode 100644 index 000000000..b6f1f2960 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + integer = pynutil.delete("integer:") + delete_space + pynutil.delete('"') + integer += pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + self.fst = self.delete_tokens(integer).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/roman.py b/nemo_text_processing/text_normalization/pl/verbalizers/roman.py new file mode 100644 index 000000000..b844768d4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/roman.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class RomanFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + name = pynutil.delete("name:") + delete_space + pynutil.delete('"') + name += pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + self.fst = self.delete_tokens(name).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/time.py b/nemo_text_processing/text_normalization/pl/verbalizers/time.py new file mode 100644 index 000000000..3a424be64 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/time.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class TimeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + value = pynini.closure(NEMO_NOT_QUOTE, 1) + hours = pynutil.delete('hours: "') + value + pynutil.delete('"') + minutes = pynini.closure( + delete_space + pynutil.insert(" ") + pynutil.delete('minutes: "') + value + pynutil.delete('"'), + 0, + 1, + ) + seconds = pynini.closure( + delete_space + pynutil.insert(" ") + pynutil.delete('seconds: "') + value + pynutil.delete('"'), + 0, + 1, + ) + self.fst = self.delete_tokens(hours + minutes + seconds).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py new file mode 100644 index 000000000..cb7020e00 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.pl.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.pl.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.pl.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.roman import RomanFst +from nemo_text_processing.text_normalization.pl.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + self.fst = ( + CardinalFst(deterministic=deterministic).fst + | OrdinalFst(deterministic=deterministic).fst + | RomanFst(deterministic=deterministic).fst + | DateFst(deterministic=deterministic).fst + | MeasureFst(deterministic=deterministic).fst + | TimeFst(deterministic=deterministic).fst + | AbbreviationFst(deterministic=deterministic).fst + ) diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py new file mode 100644 index 000000000..04b2a403e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py @@ -0,0 +1,52 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.pl.verbalizers.verbalize import VerbalizeFst + + +class VerbalizeFinalFst(GraphFst): + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"pl_tn_{deterministic}_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + return + + types = VerbalizeFst(deterministic=deterministic).fst | WordFst(deterministic=deterministic).fst + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + self.fst = (delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space).optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/tests/nemo_text_processing/pl/__init__.py b/tests/nemo_text_processing/pl/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/tests/nemo_text_processing/pl/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/pl/data_text_normalization/__init__.py b/tests/nemo_text_processing/pl/data_text_normalization/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..524b3d379 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,15 @@ +0~zero +1~jeden +2~dwa +11~jedenaście +21~dwadzieścia jeden +22~dwadzieścia dwa +105~sto pięć +999~dziewięćset dziewięćdziesiąt dziewięć +1000~tysiąc +2000~dwa tysiące +5000~pięć tysięcy +21000~dwadzieścia jeden tysięcy +1000000~milion +2000001~dwa miliony jeden +1832~tysiąc osiemset trzydzieści dwa diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..875701f16 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt @@ -0,0 +1,6 @@ +10 lutego~dziesiątego lutego +28 lutego 1832~dwudziestego ósmego lutego tysiąc osiemset trzydziestego drugiego +29.01.1991~dwudziestego dziewiątego stycznia tysiąc dziewięćset dziewięćdziesiątego pierwszego +7.VIII.2026~siódmego sierpnia dwa tysiące dwudziestego szóstego +29 listopada 1877r.~dwudziestego dziewiątego listopada tysiąc osiemset siedemdziesiątego siódmego roku +29 listopada 1877 r.~dwudziestego dziewiątego listopada tysiąc osiemset siedemdziesiątego siódmego roku diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..caff6405d --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,6 @@ +1 m~jeden metr +2 cm~dwa centymetry +16 Hz~szesnaście herców +20 kHz~dwadzieścia kiloherców +35cm~trzydzieści pięć centymetrów +1000 mm~tysiąc milimetrów diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt new file mode 100644 index 000000000..7f72da5ad --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt @@ -0,0 +1,27 @@ +~Mam 22 koty +Mam dwadzieścia dwa koty +Mam dwadzieścia dwie koty +Mam dwudziestu dwóch koty +~To był 21. test +To był dwudziesty pierwszy test +To był dwudziesta pierwsza test +To był dwudziestego pierwszego test +~tzw. +tak zwany +tak zwanego +tak zwana +tak zwaną +~ur. +urodzony +urodzona +urodzonego +urodzeni +~ang. +angielski +angielska +angielskiego +angielscy +~z ang. +z angielskiego +~do 35 cm +do trzydziestu pięciu centymetrów diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..f9b59ce0a --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,9 @@ +1.~pierwszy +2.~drugi +11.~jedenasty +20.~dwudziesty +21.~dwudziesty pierwszy +100.~setny +101.~sto pierwszy +121.~sto dwudziesty pierwszy +999.~dziewięćset dziewięćdziesiąty dziewiąty diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..c86a57bc4 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,18 @@ +Ludwik XIV~Ludwik czternasty +Ludwika XIII~Ludwika trzynastego +o Ludwiku XVIII~o Ludwiku osiemnastym +dla Ludwika XV~dla Ludwika piętnastego +z Karolem V~z Karolem piątym +przez Ludwika XIV~przez Ludwika czternastego +Ludwikiem XIV~Ludwikiem czternastym +Henryka II~Henryka drugiego +Karol X~Karol dziesiąty +Karola X~Karola dziesiątego +Karolem V~Karolem piątym +Jerzy IV~Jerzy czwarty +Napoleona III~Napoleona trzeciego +Katarzyna II~Katarzyna druga +Papież Klemens VII~Papież Klemens siódmy +o Klemensie VII~o Klemensie siódmym +Franciszka Józefa I~Franciszka Józefa pierwszego +Jana Pawła II~Jana Pawła drugiego diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..cc0716b8d --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt @@ -0,0 +1,5 @@ +o 4.25~o czwartej dwadzieścia pięć +o godz. 21:14~o godzinie dwudziestej pierwszej czternaście +4:25~czwarta dwadzieścia pięć +04:25:07~czwarta dwadzieścia pięć siedem +00:00:00~zero zero zero diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..7e33c1681 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,12 @@ +n.p.m.~nad poziomem morza +tj.~to jest +np.~na przykład +itd.~i tak dalej +m.in.~między innymi +im.~imienia +s-ka~spółka +s-ki~spółki +s-ce~spółce +ur.~urodzony +ang.~angielski +tzw.~tak zwany diff --git a/tests/nemo_text_processing/pl/test_abbreviation.py b/tests/nemo_text_processing/pl/test_abbreviation.py new file mode 100644 index 000000000..8f9664f69 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_abbreviation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from nemo_text_processing.text_normalization.pl.taggers.abbreviation import AbbreviationFst + + +class TestAbbreviation: + normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_graph(self): + abbreviation = AbbreviationFst(deterministic=False) + assert 'abbreviation { value: "A B C" }' in rewrite.top_rewrites("ABC", abbreviation.fst, 10) + assert 'abbreviation { value: "A. B. C." }' in rewrite.top_rewrites("A.B.C.", abbreviation.fst, 10) + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_audio_lattice(self): + predictions = self.normalizer.normalize("Kod ABC", n_tagged=20, punct_post_process=False) + assert "Kod A B C" in predictions diff --git a/tests/nemo_text_processing/pl/test_audio_normalizer.py b/tests/nemo_text_processing/pl/test_audio_normalizer.py new file mode 100644 index 000000000..f4a19fdb9 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_audio_normalizer.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +import pytest + +from nemo_text_processing.text_normalization.pl.taggers.tokenize_and_classify_with_audio import ClassifyFst + + +class TestAudioNormalizer: + normalizer = ClassifyFst(input_case="cased") + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_ngram_acceptor_selects_inflection(self): + masculine = pynini.accep("Mam dwadzieścia dwa koty", weight=1) + feminine = pynini.accep("Mam dwadzieścia dwie koty", weight=0) + language_model = (masculine | feminine).optimize() + assert self.normalizer.normalize("Mam 22 koty", language_model) == "Mam dwadzieścia dwie koty" + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_ngram_acceptor_selects_compound_tokenization(self): + joined = pynini.accep("To dwudziestodwulatka", weight=1) + spaced = pynini.accep("To dwudziesto dwu latka", weight=0) + language_model = (joined | spaced).optimize() + assert self.normalizer.normalize("To 22-latka", language_model) == "To dwudziesto dwu latka" diff --git a/tests/nemo_text_processing/pl/test_cardinal.py b/tests/nemo_text_processing/pl/test_cardinal.py new file mode 100644 index 000000000..5884e044c --- /dev/null +++ b/tests/nemo_text_processing/pl/test_cardinal.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst + +from ..utils import parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_cardinal.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_inflectional_graphs(self): + cardinal = CardinalFst() + assert rewrite.one_top_rewrite("1", cardinal.graphs["f_sg_nom"]) == "jedna" + assert rewrite.one_top_rewrite("2", cardinal.graphs["mp_pl_nom"]) == "dwaj" + assert rewrite.one_top_rewrite("22", cardinal.graphs["mp_pl_nom"]) == "dwudziestu dwóch" + assert rewrite.one_top_rewrite("22", cardinal.graphs["f_pl_nom"]) == "dwadzieścia dwie" + assert rewrite.one_top_rewrite("22", cardinal.graphs["pl_gen"]) == "dwudziestu dwóch" + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_compound_graph(self): + cardinal = CardinalFst() + assert rewrite.one_top_rewrite("22", cardinal.graphs["compound"]) == "dwudziestodwu" + assert rewrite.one_top_rewrite("22-latka", cardinal.compound) == "dwudziestodwulatka" + + cardinal = CardinalFst(deterministic=False) + alternatives = rewrite.top_rewrites("22", cardinal.graphs["compound"], 10) + assert "dwudziestodwu" in alternatives + assert "dwudziesto dwu" in alternatives + alternatives = rewrite.top_rewrites("22-latka", cardinal.compound, 20) + assert "dwudziestodwulatka" in alternatives + assert "dwudziesto dwu latka" in alternatives diff --git a/tests/nemo_text_processing/pl/test_date.py b/tests/nemo_text_processing/pl/test_date.py new file mode 100644 index 000000000..2a65dc71f --- /dev/null +++ b/tests/nemo_text_processing/pl/test_date.py @@ -0,0 +1,56 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.taggers.date import DateFst +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst + +from ..utils import parse_test_case_file + + +class TestDate: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_date.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_case_coordinated_graphs(self): + cardinal = CardinalFst(deterministic=True) + ordinal = OrdinalFst(deterministic=True) + date = DateFst(cardinal, ordinal, deterministic=True) + assert "mi_sg_nom" in date.graphs + assert "mi_sg_gen" in date.graphs + assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_nom"]) == ('day: "dziesiąty" month: "lutego"') + assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_gen"]) == ('day: "dziesiątego" month: "lutego"') + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_bare_year_is_only_a_nondeterministic_date(self): + cardinal = CardinalFst(deterministic=False) + ordinal = OrdinalFst(deterministic=False) + date = DateFst(cardinal, ordinal, deterministic=False) + assert rewrite.one_top_rewrite("1832", date.year_graphs["mi_sg_gen"]) == ( + 'year: "tysiąc osiemset trzydziestego drugiego"' + ) diff --git a/tests/nemo_text_processing/pl/test_measure.py b/tests/nemo_text_processing/pl/test_measure.py new file mode 100644 index 000000000..834937d1e --- /dev/null +++ b/tests/nemo_text_processing/pl/test_measure.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import parse_test_case_file + + +class TestMeasure: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_measure.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected diff --git a/tests/nemo_text_processing/pl/test_normalization_with_audio.py b/tests/nemo_text_processing/pl/test_normalization_with_audio.py new file mode 100644 index 000000000..ba9fb243d --- /dev/null +++ b/tests/nemo_text_processing/pl/test_normalization_with_audio.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + +from ..utils import get_test_cases_multiple + + +class TestNormalizeWithAudio: + normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(get_test_cases_multiple("pl/data_text_normalization/test_cases_normalize_with_audio.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_normalization_with_audio(self, test_input, expected): + predictions = self.normalizer.normalize(test_input, n_tagged=100, punct_post_process=False) + for option in expected: + assert option in predictions diff --git a/tests/nemo_text_processing/pl/test_ordinal.py b/tests/nemo_text_processing/pl/test_ordinal.py new file mode 100644 index 000000000..dd356bc0f --- /dev/null +++ b/tests/nemo_text_processing/pl/test_ordinal.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst + +from ..utils import parse_test_case_file + + +class TestOrdinal: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_ordinal.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_inflectional_graphs(self): + ordinal = OrdinalFst() + assert rewrite.one_top_rewrite("21", ordinal.graphs["f_sg_nom"]) == "dwudziesta pierwsza" + assert rewrite.one_top_rewrite("21", ordinal.graphs["mi_sg_gen"]) == "dwudziestego pierwszego" diff --git a/tests/nemo_text_processing/pl/test_roman.py b/tests/nemo_text_processing/pl/test_roman.py new file mode 100644 index 000000000..7ccf22314 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_roman.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.taggers.roman import RomanFst + +from ..utils import parse_test_case_file + + +class TestRoman: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_roman.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_two_component_name_inflection(self): + roman = RomanFst(OrdinalFst()) + assert rewrite.one_top_rewrite("Franciszka Józefa I", roman.graphs["mp_sg_gen"]) == ( + "Franciszka Józefa pierwszego" + ) diff --git a/tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..17870bc0c --- /dev/null +++ b/tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} + +runtest() { + input=$1 + echo "INPUT is $input" + cd "${GRAMMARS_DIR}" || return 1 + + while read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + denorm_pred=$(echo "$written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/ / /g' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_cardinal.txt" +} + +testTNDate() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_date.txt" +} + +testTNMeasure() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_measure.txt" +} + +testTNOrdinal() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_ordinal.txt" +} + +testTNRoman() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_roman.txt" +} + +testTNTime() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_time.txt" +} + +testTNWhitelist() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_whitelist.txt" +} + +. "$PROJECT_DIR/../shunit2/shunit2" diff --git a/tests/nemo_text_processing/pl/test_time.py b/tests/nemo_text_processing/pl/test_time.py new file mode 100644 index 000000000..61797dc3f --- /dev/null +++ b/tests/nemo_text_processing/pl/test_time.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import parse_test_case_file + + +class TestTime: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_time.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected diff --git a/tests/nemo_text_processing/pl/test_whitelist.py b/tests/nemo_text_processing/pl/test_whitelist.py new file mode 100644 index 000000000..732d87aa1 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_whitelist.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst + +from ..utils import parse_test_case_file + + +class TestWhitelist: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + audio_normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_whitelist.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_inflected_graphs_are_keyed_by_slot(self): + whitelist = WhiteListFst(input_case="cased", deterministic=True) + assert rewrite.one_top_rewrite("s-ka", whitelist.inflected_graphs["sg_nom"]) == "spółka" + assert rewrite.one_top_rewrite("s-ki", whitelist.inflected_graphs["sg_gen"]) == "spółki" + assert rewrite.one_top_rewrite("s-ce", whitelist.inflected_graphs["sg_loc"]) == "spółce" + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_ambiguous_entry_has_all_singular_forms(self): + whitelist = WhiteListFst(input_case="cased", deterministic=False) + expected = { + "sg_nom": "rok", + "sg_gen": "roku", + "sg_dat": "rokowi", + "sg_acc": "rok", + "sg_ins": "rokiem", + "sg_loc": "roku", + "sg_voc": "roku", + } + for slot, spoken in expected.items(): + assert rewrite.one_top_rewrite("r.", whitelist.nondeterministic_graphs[slot]) == spoken diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..e55704666 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -95,6 +95,7 @@ def parse_args(): "ru", 'fr', 'hu', + 'pl', 'sv', 'vi', 'zh', @@ -240,6 +241,11 @@ def parse_args(): ClassifyFst as TNClassifyFst, ) from nemo_text_processing.text_normalization.sv.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'pl': + from nemo_text_processing.text_normalization.pl.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.pl.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'vi': from nemo_text_processing.inverse_text_normalization.vi.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst,