Fix for nv-bug 4786175 (#213)

* Implements the fix

Signed-off-by: Simon Zuberek <szuberek@nvidia.com>

* Updates test cases for MONEY

Signed-off-by: Simon Zuberek <szuberek@nvidia.com>

* Updates the cache

Signed-off-by: Simon Zuberek <szuberek@nvidia.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: Simon Zuberek <szuberek@nvidia.com>
Co-authored-by: Simon Zuberek <szuberek@nvidia.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: zoobereq

Jenkinsfile

@@ -13,7 +13,7 @@ pipeline {
 
     AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-24-24-0'
     DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-03-24-0'
-    EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-15-24-0'
+    EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-20-24-0'
     ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-14-24-0'
     ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-14-24-0'
     FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-04-24-0'

nemo_text_processing/text_normalization/en/data/measure/unit.tsv

@@ -124,4 +124,12 @@ ps	PS
 s	S
 tb	TB
 tb	YB
-zb	ZB
+zb	ZB
+sec	second
+min	minute
+hr	hour
+d	day
+wk	week
+mo	month
+yr	year
+svc	service

nemo_text_processing/text_normalization/en/data/money/per_unit.tsv

@@ -1,2 +1,11 @@
 /ea	each
-/dozen
+/dozen	per dozen
+/sec	per second
+/min	per minute
+/hr	per hour
+/h	per hour
+/d	per day
+/wk	per week
+/mo	per month
+/yr	per year
+/svc	per service

nemo_text_processing/text_normalization/en/taggers/money.py

@@ -29,11 +29,12 @@
 min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv"))
 min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv"))
 maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv")))
+per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv"))
 
 
 class MoneyFst(GraphFst):
     """
-    Finite state transducer for classifying money, suppletive aware, e.g. 
+    Finite state transducer for classifying money, suppletive aware, e.g.
         $12.05 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true }
         $12.0500 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true }
         $1 -> money { currency_maj: "dollar" integer_part: "one" }
@@ -59,14 +60,14 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool =
         maj_unit_plural = convert_space(maj_singular @ SINGULAR_TO_PLURAL)
         maj_unit_singular = convert_space(maj_singular)
 
-        graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_unit_singular + pynutil.insert("\"")
-        graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_unit_plural + pynutil.insert("\"")
+        graph_maj_singular = pynutil.insert('currency_maj: "') + maj_unit_singular + pynutil.insert('"')
+        graph_maj_plural = pynutil.insert('currency_maj: "') + maj_unit_plural + pynutil.insert('"')
 
         optional_delete_fractional_zeros = pynini.closure(
             pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), 0, 1
         )
 
-        graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "one") + pynutil.insert("\"")
+        graph_integer_one = pynutil.insert('integer_part: "') + pynini.cross("1", "one") + pynutil.insert('"')
         # only for decimals where third decimal after comma is non-zero or with quantity
         decimal_delete_last_zeros = (
             pynini.closure(NEMO_DIGIT | pynutil.delete(","))
@@ -81,9 +82,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool =
             graph_maj_plural + insert_space + (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final
         )
 
-        graph_integer = (
-            pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"")
-        )
+        graph_integer = pynutil.insert('integer_part: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert('"')
 
         graph_integer_only = graph_maj_singular + insert_space + graph_integer_one
         graph_integer_only |= graph_maj_plural + insert_space + graph_integer
@@ -101,8 +100,8 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool =
             | ((NEMO_DIGIT - "0") + NEMO_DIGIT)
         )
 
-        graph_min_singular = pynutil.insert(" currency_min: \"") + min_singular + pynutil.insert("\"")
-        graph_min_plural = pynutil.insert(" currency_min: \"") + min_plural + pynutil.insert("\"")
+        graph_min_singular = pynutil.insert(' currency_min: "') + min_singular + pynutil.insert('"')
+        graph_min_plural = pynutil.insert(' currency_min: "') + min_plural + pynutil.insert('"')
         # format ** dollars ** cent
         decimal_graph_with_minor = None
         integer_graph_reordered = None
@@ -113,19 +112,19 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool =
             integer_plus_maj |= graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular
 
             integer_plus_maj_with_comma = pynini.compose(
-                NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | pynutil.delete(",")), integer_plus_maj
+                NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | pynutil.delete(",")), integer_plus_maj,
             )
             integer_plus_maj = pynini.compose(pynini.closure(NEMO_DIGIT) - "0", integer_plus_maj)
             integer_plus_maj |= integer_plus_maj_with_comma
 
             graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "one")
-            graph_fractional_one = pynutil.insert("fractional_part: \"") + graph_fractional_one + pynutil.insert("\"")
+            graph_fractional_one = pynutil.insert('fractional_part: "') + graph_fractional_one + pynutil.insert('"')
             graph_fractional = (
                 two_digits_fractional_part
                 @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1")
                 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit
             )
-            graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"")
+            graph_fractional = pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"')
 
             fractional_plus_min = graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural
             fractional_plus_min |= (
@@ -138,9 +137,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool =
                 decimal_graph_with_minor_curr |= pynutil.add_weight(
                     integer_plus_maj
                     + pynini.cross(".", " ")
-                    + pynutil.insert("fractional_part: \"")
+                    + pynutil.insert('fractional_part: "')
                     + two_digits_fractional_part @ cardinal.graph_hundred_component_at_least_one_none_zero_digit
-                    + pynutil.insert("\""),
+                    + pynutil.insert('"'),
                     weight=0.0001,
                 )
                 default_fraction_graph = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final
@@ -183,11 +182,16 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool =
         # weight for SH
         final_graph |= pynutil.add_weight(decimal_graph_with_minor, -0.0001)
 
+        # utilizes morphosyntactic features to append "per units"
+        graph_per_units = pynutil.insert(' morphosyntactic_features: "') + per_units + pynutil.insert('"')
+
         if not deterministic:
             final_graph |= integer_graph_reordered | decimal_default_reordered
             # to handle "$2.00" cases
             final_graph |= pynini.compose(
-                NEMO_SIGMA + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), integer_graph_reordered
+                NEMO_SIGMA + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), integer_graph_reordered,
             )
+
+        final_graph += graph_per_units.ques
         final_graph = self.add_tokens(final_graph.optimize())
         self.fst = final_graph.optimize()

nemo_text_processing/text_normalization/en/verbalizers/money.py

@@ -22,6 +22,10 @@
     delete_preserve_order,
 )
 
+from nemo_text_processing.text_normalization.en.utils import get_abs_path
+
+per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv"))
+
 
 class MoneyFst(GraphFst):
     """
@@ -37,11 +41,11 @@ class MoneyFst(GraphFst):
     def __init__(self, decimal: GraphFst, deterministic: bool = True):
         super().__init__(name="money", kind="verbalize", deterministic=deterministic)
         keep_space = pynini.accep(" ")
-        maj = pynutil.delete("currency_maj: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
-        min = pynutil.delete("currency_min: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        min = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')
 
         fractional_part = (
-            pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+            pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')
         )
 
         integer_part = decimal.integer
@@ -68,5 +72,14 @@ def __init__(self, decimal: GraphFst, deterministic: bool = True):
         if not deterministic:
             graph |= graph_integer + delete_preserve_order
 
+        per_units_normalized = pynini.project(per_units, "output")
+        remove_per_units_normalized = (
+            pynutil.delete(' morphosyntactic_features: "')
+            + pynutil.insert(" ")
+            + per_units_normalized
+            + pynutil.delete('" ')
+        )
+        graph += remove_per_units_normalized.ques
+
         delete_tokens = self.delete_tokens(graph)
         self.fst = delete_tokens.optimize()

tests/nemo_text_processing/en/data_text_normalization/test_cases_money.txt

@@ -64,3 +64,8 @@ $1,234.123~one thousand two hundred and thirty four point one two three dollars
 US $76.3 trillion~US seventy six point three trillion dollars
 US$76.3 trillion~seventy six point three trillion us dollars
 The price for each canned salmon is $5 , each bottle of peanut butter is $3~The price for each canned salmon is five dollars , each bottle of peanut butter is three dollars
+$20/mo is what we are currently charging.~twenty dollars per month is what we are currently charging.
+$350/yr is the fee.~three hundred and fifty dollars per year is the fee.
+£10/wk sounds good to us.~ten pounds per week sounds good to us.
+$1/d is the new subscription cost.~one dollar per day is the new subscription cost.
+$0.5/hr is the total cost.~fifty cents per hour is the total cost.